Merge branch 'oneclick_nn_training' into 'master'

Oneclick nn training

See merge request !22

.gitlab-ci.yml

@@ -43,7 +43,7 @@ integrationMXNetJobLinux:
 
 integrationCaffe2JobLinux:
   stage: linux
-  image: registry.git.rwth-aachen.de/monticore/embeddedmontiarc/generators/emadl2cpp/integrationtests/caffe2:v0.0.3
+  image: registry.git.rwth-aachen.de/monticore/embeddedmontiarc/generators/emadl2cpp/integrationtests/caffe2:v0.0.5
   script:
   - mvn -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -B  clean install --settings settings.xml -Dtest=IntegrationCaffe2Test
 

README.md

@@ -79,7 +79,8 @@ configuration VGG16{
     * `-o`    output path
     * `-b`    backend
     * `-p`   path to python (Not mandatory; Default is `/usr/bin/python`)
-    * `-f`   forced training (Not mandatory; values can be `y` for a forced training and `n` for a skip (a forced no-training)). By default, the hash value (from the training and test data, the structure of the model (.emadl) and the training parameters (.cnnt) of the model) will be compared. The model is retrained only if the hash changes. This can be used to distribute trained models, by distributing the corresponding `.training_hash` file as well, which will prevent a retraining.
+    * `-f`   forced training (Not mandatory; values can be `y` for a forced training and `n` for a skip (a forced no-training)). By default, the hash value (from the training and test data, the structure of the model (.emadl) and the training parameters (.cnnt) of the model) will be compared. The model is retrained only if the hash changes. This can be used to distribute trained models, by distributing the corresponding `.training_hash` file as well, which will prevent a retraining
+    * `-c`    compiling of generated c code (Not mandators; Default is `y`). Disable by setting to `n` when running on Windows, or on machines without `make` and `cmake` commands
 
     Assuming both the architecture definition `VGG16.emadl` and the corresponding training configuration `VGG16.cnnt` are located in a folder `models` and the target code should be generated in  a `target` folder using the `MXNet` backend, an example of a command is then:  
     ```java -jar embedded-montiarc-emadl-generator-0.2.10-jar-with-dependencies.jar -m models -r VGG16 -o target -b MXNET```

pom.xml

@@ -8,7 +8,7 @@
 
   <groupId>de.monticore.lang.monticar</groupId>
   <artifactId>embedded-montiarc-emadl-generator</artifactId>
-  <version>0.2.12</version>
+  <version>0.2.13</version>
 
   <!-- == PROJECT DEPENDENCIES ============================================= -->
 
@@ -18,7 +18,7 @@
     <emadl.version>0.2.6</emadl.version>
     <CNNTrain.version>0.2.6</CNNTrain.version>
     <cnnarch-mxnet-generator.version>0.2.14-SNAPSHOT</cnnarch-mxnet-generator.version>
-    <cnnarch-caffe2-generator.version>0.2.10-SNAPSHOT</cnnarch-caffe2-generator.version>
+    <cnnarch-caffe2-generator.version>0.2.11-SNAPSHOT</cnnarch-caffe2-generator.version>
     <embedded-montiarc-math-opt-generator>0.1.4</embedded-montiarc-math-opt-generator>
 
         <!-- .. Libraries .................................................. -->

src/main/java/de/monticore/lang/monticar/emadl/generator/EMADLGenerator.java

@@ -108,7 +108,7 @@ public class EMADLGenerator {
         return emamGen;
     }
 
-    public void generate(String modelPath, String qualifiedName, String pythonPath, String forced) throws IOException, TemplateException {
+    public void generate(String modelPath, String qualifiedName, String pythonPath, String forced, boolean doCompile) throws IOException, TemplateException {
         setModelsPath( modelPath );
         TaggingResolver symtab = EMADLAbstractSymtab.createSymTabAndTaggingResolver(getModelsPath());
         EMAComponentSymbol component = symtab.<EMAComponentSymbol>resolve(qualifiedName, EMAComponentSymbol.KIND).orElse(null);
@@ -126,24 +126,26 @@ public class EMADLGenerator {
 
 
         generateFiles(symtab, instance, symtab, pythonPath, forced);
-        try{
-          executeCommands();
-        }catch(Exception e){
-          System.out.println(e);
+        
+        if (doCompile) {
+            compile();
         }
     }
 
-    public void executeCommands() throws IOException {
-
+    public void compile() throws IOException {
         File tempScript = createTempScript();
-
         try {
             ProcessBuilder pb = new ProcessBuilder("bash", tempScript.toString());
             pb.inheritIO();
             Process process = pb.start();
-          process.waitFor();
+            int returnCode = process.waitFor();
+            if(returnCode != 0) {
+                Log.error("During compilation, an error occured. See above for more details.");
+                System.exit(1);
+            }
         }catch(Exception e){
-          System.out.println(e);
+            Log.error("During compilation, the following error occured: '" + e.toString() + "'");
+            System.exit(1);
         } finally {
             tempScript.delete();
         }
@@ -252,15 +254,13 @@ public class EMADLGenerator {
                         exitCode = p.waitFor();
                     }
                     catch(InterruptedException e) {
-                        //throw new Exception("Error: Training aborted" + e.toString());
-                        System.out.println("Error: Training aborted" + e.toString());
-                        continue;
+                        Log.error("Training aborted: exit code " + Integer.toString(exitCode));
+                        System.exit(1);
                     }
 
                     if(exitCode != 0) {
-                        //throw new Exception("Error: Training error");
-                        System.out.println("Error: Training failed" + Integer.toString(exitCode));
-                        continue;
+                        Log.error("Training failed: exit code " + Integer.toString(exitCode));
+                        System.exit(1);
                     }
 
                     fileContentsTrainingHashes.add(new FileContent(trainingHash, componentConfigFilename + ".training_hash"));

src/main/java/de/monticore/lang/monticar/emadl/generator/EMADLGeneratorCli.java

@@ -71,6 +71,13 @@ public class EMADLGeneratorCli {
             .required(false)
             .build();
 
+    public static final Option OPTION_COMPILE = Option.builder("c")
+            .longOpt("compile")
+            .desc("Compile the generated c code. Needs to be disabled eg. on Windows. Options: y (compile), n (don't compile). Default is y")
+            .hasArg(true)
+            .required(false)
+            .build();
+
 
     private EMADLGeneratorCli() {
     }
@@ -92,6 +99,7 @@ public class EMADLGeneratorCli {
         options.addOption(OPTION_BACKEND);
         options.addOption(OPTION_RESTRAINED_TRAINING);
         options.addOption(OPTION_TRAINING_PYTHON_PATH);
+        options.addOption(OPTION_COMPILE);
         return options;
     }
 
@@ -113,8 +121,10 @@ public class EMADLGeneratorCli {
         String backendString = cliArgs.getOptionValue(OPTION_BACKEND.getOpt());
         String forced = cliArgs.getOptionValue(OPTION_RESTRAINED_TRAINING.getOpt());
         String pythonPath = cliArgs.getOptionValue(OPTION_TRAINING_PYTHON_PATH.getOpt());
+        String compile = cliArgs.getOptionValue(OPTION_COMPILE.getOpt());
         final String DEFAULT_BACKEND = "MXNET";
         final String DEFAULT_FORCED = "UNSET";
+        final String DEFAULT_COMPILE = "y";
 
         if (backendString == null) {
             Log.warn("backend not specified. backend set to default value " + DEFAULT_BACKEND);
@@ -138,13 +148,22 @@ public class EMADLGeneratorCli {
             Log.error("specified setting ("+forced+") for forcing/preventing training not supported. set to default value " + DEFAULT_FORCED);
             forced = DEFAULT_FORCED;
         }
+
         EMADLGenerator generator = new EMADLGenerator(backend.get());
 
+        if (compile == null) {
+            compile = DEFAULT_COMPILE;
+        }
+        else if(!compile.equals("y") && !compile.equals("n")) {
+            Log.error("specified setting ("+compile+") for skipping the compilation not supported. set to default value " + DEFAULT_COMPILE);
+            compile = DEFAULT_COMPILE;
+        }
+
         if (outputPath != null){
             generator.setGenerationTargetPath(outputPath);
         }
         try{
-            generator.generate(cliArgs.getOptionValue(OPTION_MODELS_PATH.getOpt()), rootModelName, pythonPath, forced);
+            generator.generate(cliArgs.getOptionValue(OPTION_MODELS_PATH.getOpt()), rootModelName, pythonPath, forced, compile.equals("y"));
         }
         catch (IOException e){
             Log.error("io error during generation", e);

src/test/java/de/monticore/lang/monticar/emadl/GenerationTest.java

@@ -51,7 +51,7 @@ public class GenerationTest extends AbstractSymtabTest {
     @Test
     public void testCifar10Generation() throws IOException, TemplateException {
         Log.getFindings().clear();
-        String[] args = {"-m", "src/test/resources/models/", "-r", "cifar10.Cifar10Classifier", "-b", "MXNET", "-f", "n"};
+        String[] args = {"-m", "src/test/resources/models/", "-r", "cifar10.Cifar10Classifier", "-b", "MXNET", "-f", "n", "-c", "n"};
         EMADLGeneratorCli.main(args);
         assertTrue(Log.getFindings().isEmpty());
 
@@ -73,7 +73,7 @@ public class GenerationTest extends AbstractSymtabTest {
     @Test
     public void testSimulatorGeneration() throws IOException, TemplateException {
         Log.getFindings().clear();
-        String[] args = {"-m", "src/test/resources/models/", "-r", "simulator.MainController", "-b", "MXNET", "-f", "n"};
+        String[] args = {"-m", "src/test/resources/models/", "-r", "simulator.MainController", "-b", "MXNET", "-f", "n", "-c", "n"};
         EMADLGeneratorCli.main(args);
         assertTrue(Log.getFindings().isEmpty());
     }
@@ -81,7 +81,7 @@ public class GenerationTest extends AbstractSymtabTest {
     @Test
     public void testAddGeneration() throws IOException, TemplateException {
         Log.getFindings().clear();
-        String[] args = {"-m", "src/test/resources/models/", "-r", "Add", "-b", "MXNET", "-f", "n"};
+        String[] args = {"-m", "src/test/resources/models/", "-r", "Add", "-b", "MXNET", "-f", "n", "-c", "n"};
         EMADLGeneratorCli.main(args);
         assertTrue(Log.getFindings().isEmpty());
     }
@@ -89,7 +89,7 @@ public class GenerationTest extends AbstractSymtabTest {
     @Test
     public void testAlexnetGeneration() throws IOException, TemplateException {
         Log.getFindings().clear();
-        String[] args = {"-m", "src/test/resources/models/", "-r", "Alexnet", "-b", "MXNET", "-f", "n"};
+        String[] args = {"-m", "src/test/resources/models/", "-r", "Alexnet", "-b", "MXNET", "-f", "n", "-c", "n"};
         EMADLGeneratorCli.main(args);
         assertTrue(Log.getFindings().isEmpty());
     }
@@ -97,7 +97,7 @@ public class GenerationTest extends AbstractSymtabTest {
     @Test
     public void testResNeXtGeneration() throws IOException, TemplateException {
         Log.getFindings().clear();
-        String[] args = {"-m", "src/test/resources/models/", "-r", "ResNeXt50", "-b", "MXNET", "-f", "n"};
+        String[] args = {"-m", "src/test/resources/models/", "-r", "ResNeXt50", "-b", "MXNET", "-f", "n", "-c", "n"};
         EMADLGeneratorCli.main(args);
         assertTrue(Log.getFindings().isEmpty());
     }
@@ -105,7 +105,7 @@ public class GenerationTest extends AbstractSymtabTest {
     @Test
     public void testThreeInputGeneration() throws IOException, TemplateException {
         Log.getFindings().clear();
-        String[] args = {"-m", "src/test/resources/models/", "-r", "ThreeInputCNN_M14", "-b", "MXNET", "-f", "n"};
+        String[] args = {"-m", "src/test/resources/models/", "-r", "ThreeInputCNN_M14", "-b", "MXNET", "-f", "n", "-c", "n"};
         EMADLGeneratorCli.main(args);
         assertTrue(Log.getFindings().size() == 1);
     }
@@ -113,7 +113,7 @@ public class GenerationTest extends AbstractSymtabTest {
     @Test
     public void testMultipleOutputsGeneration() throws IOException, TemplateException {
         Log.getFindings().clear();
-        String[] args = {"-m", "src/test/resources/models/", "-r", "MultipleOutputs", "-b", "MXNET", "-f", "n"};
+        String[] args = {"-m", "src/test/resources/models/", "-r", "MultipleOutputs", "-b", "MXNET", "-f", "n", "-c", "n"};
         EMADLGeneratorCli.main(args);
         assertTrue(Log.getFindings().size() == 1);
     }
@@ -121,7 +121,7 @@ public class GenerationTest extends AbstractSymtabTest {
     @Test
     public void testVGGGeneration() throws IOException, TemplateException {
         Log.getFindings().clear();
-        String[] args = {"-m", "src/test/resources/models/", "-r", "VGG16", "-b", "MXNET", "-f", "n"};
+        String[] args = {"-m", "src/test/resources/models/", "-r", "VGG16", "-b", "MXNET", "-f", "n", "-c", "n"};
         EMADLGeneratorCli.main(args);
         assertTrue(Log.getFindings().isEmpty());
     }
@@ -130,7 +130,7 @@ public class GenerationTest extends AbstractSymtabTest {
     public void testMultipleInstances() throws IOException, TemplateException {
         try {
             Log.getFindings().clear();
-            String[] args = {"-m", "src/test/resources/models/", "-r", "InstanceTest.MainB", "-b", "MXNET", "-f", "n"};
+            String[] args = {"-m", "src/test/resources/models/", "-r", "InstanceTest.MainB", "-b", "MXNET", "-f", "n", "-c", "n"};
             EMADLGeneratorCli.main(args);
             assertTrue(Log.getFindings().isEmpty());
         }
@@ -142,7 +142,7 @@ public class GenerationTest extends AbstractSymtabTest {
     @Test
     public void testMnistClassifier() throws IOException, TemplateException {
         Log.getFindings().clear();
-        String[] args = {"-m", "src/test/resources/models/", "-r", "mnist.MnistClassifier", "-b", "CAFFE2", "-f", "n"};
+        String[] args = {"-m", "src/test/resources/models/", "-r", "mnist.MnistClassifier", "-b", "CAFFE2", "-f", "n", "-c", "n"};
         EMADLGeneratorCli.main(args);
         assertTrue(Log.getFindings().isEmpty());
 

src/test/resources/docker/caffe2/Dockerfile

@@ -50,10 +50,3 @@ RUN apt-get update && \
     rm -rf /var/lib/apt/lists/* 
 
 RUN python -c 'import lmdb' && echo "LMDB success" || echo "LMDB failure"
-
-RUN apt remove nvidia-*
-RUN apt update
-RUN apt-key adv --fetch-keys  http://developer.download.nvidia.com/compute/cuda/repos/ubuntu1604/x86_64/7fa2af80.pub
-RUN bash -c 'echo "deb http://developer.download.nvidia.com/compute/cuda/repos/ubuntu1604/x86_64 /" > /etc/apt/sources.list.d/cuda.list'
-RUN apt update
-RUN apt install -y cuda-toolkit-9.2
\ No newline at end of file