Skip to content

GitLab

Commit 07ebdd04 authored by Alexander David Hellwig's avatar Alexander David Hellwig
Browse files

Transform Cli from options(e.g. -g=cpp) to json

parent e6a8e572
Pipeline #95695 passed with stages
in 10 minutes and 12 seconds
Show whitespace changes
Inline Side-by-side
Showing with 288 additions and 156 deletions
README.md
View file @ 07ebdd04
......@@ -82,16 +82,23 @@ ROS Kinetic currently only supports Linux and the installation is described [her
Maven generates the jar `embedded-montiarc-math-middleware-generator-{Version}-jar-with-dependencies.jar`
and the cli is located in `de.monticore.lang.monticar.generator.middleware.DistributedTargetGeneratorCli`.
CLI Options:
* -m/--models-dir: full path to directory with EMAM models
* -r/--root-model: fully qualified name of the root model
* -o/--output-dir: full path to output directory for generated files
* -g/--generators: identifiers for the generators that should be used
* currently supported:
* cpp
* roscpp
* seperated by ','
* example: cpp,roscpp
Parameters: `${file path to config json}` OR `-r ${raw json config string}`
```
Schema of config json:
{
'modelsDir':'<path to directory with EMAM models>',
'outputDir':'<path to output directory for generated files>',
'rootModel':'<fully qualified name of the root model>',
'generators':['<identifier for first generator>', '<identifier for second generator>',...],
'emadlBackend':'<deep-learning-framework backend. Options: MXNET, CAFFE2>'
}
```
Generator Options:
- Behaviour generators:
- 'cpp': EMAM2CPP
- 'emadlcpp': EMADL2CPP
- Middleware generators:
- 'roscpp': EMAM2Roscpp
Example: [CliUsage.sh](https://git.rwth-aachen.de/monticore/EmbeddedMontiArc/generators/EMAM2Middleware/blob/master/src/test/resources/CliUsage.sh)
......
pom.xml
View file @ 07ebdd04
......@@ -9,7 +9,7 @@
<groupId>de.monticore.lang.monticar</groupId>
<artifactId>embedded-montiarc-math-middleware-generator</artifactId>
<version>0.0.12-SNAPSHOT</version>
<version>0.0.13-SNAPSHOT</version>
<!-- == PROJECT DEPENDENCIES ============================================= -->
......@@ -30,6 +30,7 @@
<guava.version>18.0</guava.version>
<junit.version>4.12</junit.version>
<logback.version>1.1.2</logback.version>
<gson.version>2.8.5</gson.version>
<!-- .. Plugins ....................................................... -->
<assembly.plugin>2.5.4</assembly.plugin>
......@@ -51,6 +52,13 @@
</properties>
<dependencies>
<!-- https://mvnrepository.com/artifact/com.google.code.gson/gson -->
<dependency>
<groupId>com.google.code.gson</groupId>
<artifactId>gson</artifactId>
<version>${gson.version}</version>
</dependency>
<dependency>
<groupId>org.jgrapht</groupId>
<artifactId>jgrapht-core</artifactId>
......@@ -119,12 +127,6 @@
<version>2.3.23</version>
</dependency>
<dependency>
<groupId>commons-cli</groupId>
<artifactId>commons-cli</artifactId>
<version>1.4</version>
</dependency>
<!-- .. Test Libraries ............................................... -->
<dependency>
<groupId>junit</groupId>
......
src/main/java/de/monticore/lang/monticar/generator/middleware/CliParameters.java 0 → 100644
View file @ 07ebdd04
package de.monticore.lang.monticar.generator.middleware;
import java.util.Set;
public class CliParameters {
private String modelsDir;
private String outputDir;
private String rootModel;
private Set<String> generators;
private String emadlBackend;
public CliParameters() {
}
public CliParameters(String modelsDir, String outputDir, String rootModel, Set<String> generators) {
this(modelsDir, outputDir, rootModel, generators, "MXNET");
}
public CliParameters(String modelsDir, String outputDir, String rootModel, Set<String> generators, String emadlBackend) {
this.modelsDir = modelsDir;
this.outputDir = outputDir;
this.rootModel = rootModel;
this.generators = generators;
this.emadlBackend = emadlBackend;
}
public String getModelsDir() {
return modelsDir;
}
public String getOutputDir() {
return outputDir;
}
public String getRootModel() {
return rootModel;
}
public Set<String> getGenerators() {
return generators;
}
public String getEmadlBackend() {
return emadlBackend;
}
}
src/main/java/de/monticore/lang/monticar/generator/middleware/DistributedTargetGeneratorCli.java
View file @ 07ebdd04
package de.monticore.lang.monticar.generator.middleware;
import com.google.gson.Gson;
import com.google.gson.stream.JsonReader;
import de.monticore.lang.embeddedmontiarc.embeddedmontiarc._symboltable.ExpandedComponentInstanceSymbol;
import de.monticore.lang.embeddedmontiarc.tagging.middleware.ros.RosToEmamTagSchema;
import de.monticore.lang.monticar.emadl.generator.EMADLAbstractSymtab;
......@@ -14,6 +16,8 @@ import de.monticore.lang.tagging._symboltable.TaggingResolver;
import de.se_rwth.commons.logging.Log;
import org.apache.commons.cli.*;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Paths;
......@@ -22,47 +26,23 @@ import java.util.HashSet;
import java.util.Set;
import java.util.stream.Collectors;
/**
* java -cp target/embedded-montiarc-math-middleware-generator-0.0.1-SNAPSHOT-jar-with-dependencies.jar \
* de.monticore.lang.monticar.generator.middleware.DistributedTargetGeneratorCli \
* --models-dir=src/test/resources/ \
* --root-model=tests.a.addComp \
* --output-dir=target/cli-test/src \
* --generators=cpp,roscpp
*/
public final class DistributedTargetGeneratorCli {
private static final Option OPTION_MODELS_PATH = Option.builder("m")
.longOpt("models-dir")
.desc("full path to directory with EMAM models")
.hasArg(true)
.required(true)
.build();
private static final Option OPTION_ROOT_MODEL = Option.builder("r")
.longOpt("root-model")
.desc("fully qualified name of the root model")
.hasArg(true)
.required(true)
.build();
private static final Option OPTION_OUTPUT_PATH = Option.builder("o")
.longOpt("output-dir")
.desc("full path to output directory for generated files")
.hasArg(true)
.required(true)
.build();
private static final Option OPTION_GENERATORS = Option.builder("g")
.longOpt("generators")
.desc("identifiers for the generators that should be used")
.hasArg(true)
.required(true)
.numberOfArgs(10)
.valueSeparator(',')
.build();
private static final Option OPTION_EMADL_BACKEND = EMADLGeneratorCli.OPTION_BACKEND;
private static final String helpString = "Parameters: <file path to config json> OR -r <raw json config string>\n\n" +
"Schema of config json:\n" +
"{\n" +
" 'modelsDir':'<path to directory with EMAM models>',\n" +
" 'outputDir':'<path to output directory for generated files>',\n" +
" 'rootModel':'<fully qualified name of the root model>',\n" +
" 'generators':['<identifier for first generator>', '<identifier for second generator>',...],\n" +
" 'emadlBackend':'<deep-learning-framework backend. Options: MXNET, CAFFE2>'\n" +
"}\n\n" +
"Generator Options:\n" +
"Behaviour generators:\n" +
" 'cpp': EMAM2CPP\n" +
" 'emadlcpp': EMADL2CPP\n" +
"Middleware generators:\n" +
" 'roscpp': EMAM2Roscpp";
public static final String GENERATOR_CPP = "cpp";
public static final String GENERATOR_EMADL = "emadlcpp";
......@@ -73,23 +53,30 @@ public final class DistributedTargetGeneratorCli {
public static void main(String[] args) {
System.out.println(Arrays.toString(args));
Gson gson = new Gson();
CliParameters parameters = null;
//File name only
if(args.length == 1){
String filePath = expandHomeDir(args[0]);
try {
JsonReader jsonReader = new JsonReader(new FileReader(filePath));
parameters = gson.fromJson(jsonReader, CliParameters.class);
} catch (FileNotFoundException e) {
Log.error("0x49E6A: Can not find specified config file: " + args[0] + "!");
}
Options options = getOptions();
CommandLineParser parser = new DefaultParser();
CommandLine cliArgs = parseArgs(options, parser, args);
if (cliArgs != null) {
runGenerator(cliArgs);
}
//raw mode
else if(args.length == 2 && args[0].equals("-r")){
parameters = gson.fromJson(args[1], CliParameters.class);
}else{
System.out.println(helpString);
}
private static Options getOptions() {
Options options = new Options();
options.addOption(OPTION_MODELS_PATH);
options.addOption(OPTION_ROOT_MODEL);
options.addOption(OPTION_OUTPUT_PATH);
options.addOption(OPTION_GENERATORS);
options.addOption(OPTION_EMADL_BACKEND);
return options;
if(parameters != null){
runGenerator(parameters);
}
}
private static Set<String> getGeneratorNames() {
......@@ -101,70 +88,72 @@ public final class DistributedTargetGeneratorCli {
return res;
}
private static CommandLine parseArgs(Options options, CommandLineParser parser, String[] args) {
CommandLine cliArgs;
try {
cliArgs = parser.parse(options, args);
} catch (ParseException e) {
Log.error("0x9A1AC: Argument parsing exception", e);
return null;
public static void runGenerator(CliParameters cliParameters) {
if(cliParameters.getModelsDir() == null){
Log.error("0xEB6FE: No models dir specified!");
return;
}
return cliArgs;
String fullModelsDirPath = expandHomeDir(cliParameters.getModelsDir());
if(cliParameters.getGenerators().size() == 0){
Log.error("0x6178E: No generator was specified!");
return;
}
private static void runGenerator(CommandLine cliArgs) {
String modelsDirPath = expandHomeDir(cliArgs.getOptionValue(OPTION_MODELS_PATH.getOpt()));
if(!Files.isDirectory(Paths.get(modelsDirPath))){
Log.error("0x6444B: Models dir does not exist: " + modelsDirPath);
if(!Files.isDirectory(Paths.get(fullModelsDirPath))){
Log.error("0x6444B: Models dir does not exist: " + fullModelsDirPath);
return;
}
String outputPath = expandHomeDir(cliArgs.getOptionValue(OPTION_OUTPUT_PATH.getOpt()));
String rootModelName = cliArgs.getOptionValue(OPTION_ROOT_MODEL.getOpt());
Set<String> generators = Arrays.stream(cliArgs.getOptionValues(OPTION_GENERATORS.getOpt())).collect(Collectors.toSet());
TaggingResolver taggingResolver;
if (generators.contains(GENERATOR_EMADL)) {
taggingResolver = EMADLAbstractSymtab.createSymTabAndTaggingResolver(modelsDirPath);
if (cliParameters.getGenerators().contains(GENERATOR_EMADL)) {
taggingResolver = EMADLAbstractSymtab.createSymTabAndTaggingResolver(fullModelsDirPath);
}
else{
taggingResolver = AbstractSymtab.createSymTabAndTaggingResolver(modelsDirPath);
taggingResolver = AbstractSymtab.createSymTabAndTaggingResolver(fullModelsDirPath);
}
DistributedTargetGenerator generator = new DistributedTargetGenerator();
generator.setGenerationTargetPath(outputPath);
generator.setGenerationTargetPath(cliParameters.getOutputDir());
Set<String> validGenNames = getGeneratorNames();
generators.forEach(genName -> {
cliParameters.getGenerators().forEach(genName -> {
if (validGenNames.contains(genName)) {
Log.warn("Using generator " + genName);
} else {
Log.error("0xE28B6: Not a valid generator Name:" + genName +".");
return;
}
});
ExpandedComponentInstanceSymbol componentInstanceSymbol = taggingResolver.<ExpandedComponentInstanceSymbol>resolve(rootModelName, ExpandedComponentInstanceSymbol.KIND).orElse(null);
ExpandedComponentInstanceSymbol componentInstanceSymbol = taggingResolver.<ExpandedComponentInstanceSymbol>resolve(cliParameters.getRootModel(), ExpandedComponentInstanceSymbol.KIND).orElse(null);
if (componentInstanceSymbol == null) {
Log.error("0x5FFAE: The given component cannot be resolved.");
return;
}
if (generators.contains(GENERATOR_CPP)) {
if (cliParameters.getGenerators().contains(GENERATOR_CPP)) {
generator.add(new CPPGenImpl(), "cpp");
}
if (generators.contains(GENERATOR_EMADL)) {
String backendString = cliArgs.getOptionValue(OPTION_EMADL_BACKEND.getOpt());
generator.add(new EMADLGeneratorImpl(cliArgs.getOptionValue(OPTION_MODELS_PATH.getOpt()), backendString), "cpp");
if (cliParameters.getGenerators().contains(GENERATOR_EMADL)) {
if(cliParameters.getEmadlBackend() != null && !cliParameters.getEmadlBackend().equals("")) {
generator.add(new EMADLGeneratorImpl(fullModelsDirPath, cliParameters.getEmadlBackend()), "cpp");
}else{
Log.error("0x0D839: Generator EMADL was specified but no backend was selected!");
return;
}
}
if (generators.contains(GENERATOR_ROSCPP)) {
if (cliParameters.getGenerators().contains(GENERATOR_ROSCPP)) {
generator.add(new RosCppGenImpl(), "roscpp");
RosToEmamTagSchema.registerTagTypes(taggingResolver);
TagHelper.resolveTags(taggingResolver, componentInstanceSymbol);
}
if (generators.contains(GENERATOR_ODV)) {
if (cliParameters.getGenerators().contains(GENERATOR_ODV)) {
generator.add(new ODVGenImpl(), "odv");
}
......@@ -173,7 +162,6 @@ public final class DistributedTargetGeneratorCli {
} catch (IOException e) {
Log.error("Error generating files", e);
}
}
private static String expandHomeDir(String path) {
......
src/test/java/de/monticore/lang/monticar/generator/middleware/CliTest.java
View file @ 07ebdd04
......@@ -2,89 +2,115 @@ package de.monticore.lang.monticar.generator.middleware;
import de.monticore.lang.embeddedmontiarc.LogConfig;
import de.se_rwth.commons.logging.Finding;
import de.se_rwth.commons.logging.Log;
import org.apache.commons.lang3.ArrayUtils;
import org.junit.*;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.List;
import java.util.stream.Collectors;
import static org.junit.Assert.assertTrue;
public class CliTest{
private static final String VALID_MODELS_DIR_OPTION = "--models-dir=src/test/resources/";
private static final String VALID_ROOT_MODEL_OPTION = "--root-model=tests.a.addComp";
private static final String VALID_GENERATOR_CPP_OPTION = "--generators=cpp";
private static final String VALID_GENERATOR_ALL_OPTION = "--generators=cpp,roscpp,odv";
public class CliTest {
private static final String VALID_MODELS_DIR_OPTION = "src/test/resources/";
private static final String VALID_ROOT_MODEL_OPTION = "tests.a.addComp";
private static final List<String> VALID_GENERATOR_CPP_OPTION = Arrays.asList("cpp");
private static final List<String> VALID_GENERATOR_ALL_OPTION = Arrays.asList("cpp", "roscpp", "odv");
private static final String INVALID_MODELS_DIR_OPTION = "--models-dir=src/invalid/resources/";
private static final String INVALID_ROOT_MODEL_OPTION = "--root-model=invalid.invalid.addComp";
private static final String INVALID_GENERATOR_OPTION = "--generators=invalid";
private static final String INVALID_GENERATOR_EMPTY_OPTION = "--generators=";
private static final String INVALID_MODELS_DIR_OPTION = "src/invalid/resources/";
private static final String INVALID_ROOT_MODEL_OPTION = "invalid.invalid.addComp";
private static final List<String> INVALID_GENERATOR_OPTION = Arrays.asList("invalid");
private static final List<String> INVALID_GENERATOR_EMPTY_OPTION = new ArrayList<>();
public static final String RESNET_MODELNAME = "tests.emadlTests.resNet34";
private String buildParameterJson(String modelsDir, String rootModel, Collection<String> generators, String outputDir) {
return buildParameterJson(modelsDir, rootModel, generators, outputDir, null);
}
private String buildParameterJson(String modelsDir, String rootModel, Collection<String> generators, String outputDir, String emadlBackend) {
String result = "{";
result += "'modelsDir': '" + modelsDir + "', ";
result += "'rootModel': '" + rootModel + "', ";
result += "'generators': [" + generators.stream().map(g -> "'" + g + "'").collect(Collectors.joining(", ")) + "], ";
if (emadlBackend != null) {
result += "'emadlBackend': '" + emadlBackend + "', ";
}
result += "'outputDir': '" + outputDir + "'";
result += "}";
return result;
}
@BeforeClass
public static void initLog(){
public static void initLog() {
LogConfig.init();
LogConfig.enableFailQuick(false);
}
@Before
public void clearFindings(){
public void clearFindings() {
LogConfig.getFindings().clear();
}
@AfterClass
public static void resetLog(){
public static void resetLog() {
LogConfig.getFindings().clear();
LogConfig.enableFailQuick(true);
}
@Test
public void testParserError(){
public void testParserError() {
String[] args = {
"invalidParsingArg"};
DistributedTargetGeneratorCli.main(args);
assertTrue(LogConfig.getErrorCount() == 1);
assertTrue(LogConfig.getFindings().stream().map(Finding::getMsg).anyMatch(msg -> msg.contains("0x9A1AC")));
assertTrue(LogConfig.getFindings().stream().map(Finding::getMsg).anyMatch(msg -> msg.contains("0x49E6A")));
}
@Test
public void testNoGenerators(){
String[] args = {
public void testNoGenerators() {
String json = buildParameterJson(
VALID_MODELS_DIR_OPTION,
VALID_ROOT_MODEL_OPTION,
INVALID_GENERATOR_EMPTY_OPTION,
"--output-dir=target/cliTest/NoGenerators/"};
DistributedTargetGeneratorCli.main(args);
"target/cliTest/NoGenerators/");
DistributedTargetGeneratorCli.main(new String[]{"-r", json});
assertTrue(LogConfig.getErrorCount() == 1);
assertTrue(LogConfig.getFindings().stream().map(Finding::getMsg).anyMatch(msg -> msg.contains("0xE28B6")));
assertTrue(LogConfig.getFindings().stream().map(Finding::getMsg).anyMatch(msg -> msg.contains("0x6178E")));
}
@Test
public void testInvalidGenerator(){
String[] args = {
public void testInvalidGenerator() {
String json = buildParameterJson(
VALID_MODELS_DIR_OPTION,
VALID_ROOT_MODEL_OPTION,
INVALID_GENERATOR_OPTION,
"--output-dir=target/cliTest/InvalidGenerators/"};
DistributedTargetGeneratorCli.main(args);
"target/cliTest/InvalidGenerators/");
DistributedTargetGeneratorCli.main(new String[]{"-r", json});
assertTrue(LogConfig.getErrorCount() == 1);
assertTrue(logContains("0xE28B6"));
}
@Test
public void testOneGenerator(){
public void testOneGenerator() {
String targetDir = "target/cliTest/OneGenerator/";
String[] args = {
String json = buildParameterJson(
VALID_MODELS_DIR_OPTION,
VALID_ROOT_MODEL_OPTION,
VALID_GENERATOR_CPP_OPTION,
"--output-dir=" + targetDir};
DistributedTargetGeneratorCli.main(args);
targetDir);
DistributedTargetGeneratorCli.main(new String[]{"-r", json});
String[] positiveFileNames = {
"CMakeLists.txt",
"tests_a_addComp/cpp/tests_a_addComp.h",
......@@ -99,15 +125,17 @@ public class CliTest{
}
@Test
public void testSingleEMADLGenerator(){
public void testSingleEMADLGenerator() {
String targetDir = "target/cliTest/SingleEmadlTest/";
String[] args = {
String json = buildParameterJson(
VALID_MODELS_DIR_OPTION,
"--root-model="+RESNET_MODELNAME,
"--generators=emadlcpp",
"--backend=MXNET",
"--output-dir=" + targetDir};
DistributedTargetGeneratorCli.main(args);
RESNET_MODELNAME,
Arrays.asList("emadlcpp"),
targetDir,
"MXNET");
DistributedTargetGeneratorCli.main(new String[]{"-r", json});
String[] positiveFileNames = getEMADLGeneratedFilesList(false);
for (String positiveFileName : positiveFileNames) {
......@@ -118,13 +146,26 @@ public class CliTest{
@Test
public void testEMADLAndRosGenerator(){
String targetDir = "target/cliTest/EmadlRosTest/";
String[] args = {
String json = buildParameterJson(
VALID_MODELS_DIR_OPTION,
"--root-model="+RESNET_MODELNAME,
"--generators=emadlcpp,roscpp",
"--backend=MXNET",
"--output-dir=" + targetDir};
DistributedTargetGeneratorCli.main(args);
RESNET_MODELNAME,
Arrays.asList("emadlcpp","roscpp"),
targetDir,
"MXNET");
DistributedTargetGeneratorCli.main(new String[]{"-r", json});
String[] positiveFileNames = getEMADLGeneratedFilesList(true);
for (String positiveFileName : positiveFileNames) {
assertTrue(Files.exists(Paths.get(targetDir + positiveFileName)));
}
}
@Test
public void testEMADLConfigFile(){
String targetDir = "target/cliTest/emadlConfigFile/";
DistributedTargetGeneratorCli.main(new String[]{"src/test/resources/config/emadl.json"});
String[] positiveFileNames = getEMADLGeneratedFilesList(true);
for (String positiveFileName : positiveFileNames) {
......@@ -132,6 +173,12 @@ public class CliTest{
}
}
@Test
public void testInvalidConfigFile(){
DistributedTargetGeneratorCli.main(new String[]{"src/test/resources/config/invalid.json"});
assertTrue(Log.getErrorCount() > 0);
}
private String[] getEMADLGeneratedFilesList(boolean generateRosFiles) {
String modelName = RESNET_MODELNAME.replace('.', '_');
String[] generatedFiles = {
......@@ -160,15 +207,36 @@ public class CliTest{
return generatedFiles;
}