Merge branch 'oneclick_nn_training' into 'master'

Oneclick nn training

See merge request !29

pom.xml

@@ -8,15 +8,15 @@
 
   <groupId>de.monticore.lang.monticar</groupId>
   <artifactId>cnnarch-caffe2-generator</artifactId>
-  <version>0.2.8-SNAPSHOT</version>
+  <version>0.2.8</version>
 
   <!-- == PROJECT DEPENDENCIES ============================================= -->
 
   <properties>
 
     <!-- .. SE-Libraries .................................................. -->
-      <CNNArch.version>0.2.7</CNNArch.version>
-      <CNNTrain.version>0.2.6-SNAPSHOT</CNNTrain.version>
+      <CNNArch.version>0.2.8</CNNArch.version>
+      <CNNTrain.version>0.2.6</CNNTrain.version>
       <embedded-montiarc-math-generator>0.1.2-SNAPSHOT</embedded-montiarc-math-generator>
     
         <!-- .. Libraries .................................................. -->

settings.xml

@@ -39,7 +39,7 @@
       <id>se-nexus</id>
       <username>cibuild</username>
       <password>${env.cibuild}</password>
-    </server>     
+    </server>
   </servers>
 
   <mirrors>
@@ -47,7 +47,7 @@
       <id>se-nexus</id>
       <mirrorOf>external:*</mirrorOf>
       <url>https://nexus.se.rwth-aachen.de/content/groups/public</url>
-    </mirror>	 
+    </mirror>
   </mirrors>
 
   <profiles>
@@ -93,5 +93,5 @@
 
   <activeProfiles>
     <activeProfile>se-nexus</activeProfile>
-  </activeProfiles>  
-</settings>
\ No newline at end of file
+  </activeProfiles>
+</settings>

src/main/java/de/monticore/lang/monticar/cnnarch/caffe2generator/CNNArch2Caffe2.java

@@ -28,6 +28,7 @@ import de.monticore.lang.monticar.cnnarch._symboltable.ArchitectureElementSymbol
 import de.monticore.lang.monticar.cnnarch._symboltable.CompositeElementSymbol;
 import de.monticore.lang.monticar.cnnarch._symboltable.CNNArchCompilationUnitSymbol;
 import de.monticore.lang.monticar.cnnarch._symboltable.CNNArchLanguage;
+import de.monticore.lang.monticar.cnnarch.DataPathConfigParser;
 import de.monticore.lang.monticar.generator.FileContent;
 import de.monticore.lang.monticar.generator.cmake.CMakeConfig;
 import de.monticore.lang.monticar.generator.cmake.CMakeFindModule;
@@ -46,6 +47,7 @@ import java.util.List;
 public class CNNArch2Caffe2 implements CNNArchGenerator{
 
     private String generationTargetPath;
+    private String modelPath;
 
     private boolean isSupportedLayer(ArchitectureElementSymbol element, LayerSupportChecker layerChecker){
         List<ArchitectureElementSymbol> constructLayerElemList;
@@ -76,6 +78,14 @@ public class CNNArch2Caffe2 implements CNNArchGenerator{
         return true;
     }
 
+    public String getModelPath(){
+        return modelPath;
+    }
+
+    public void setModelPath(Path modelPath){
+        this.modelPath = modelPath.toString();
+    }
+    
     private static void quitGeneration(){
         Log.error("Code generation is aborted");
         System.exit(1);
@@ -104,6 +114,7 @@ public class CNNArch2Caffe2 implements CNNArchGenerator{
     public void generate(Path modelsDirPath, String rootModelName){
         final ModelPath mp = new ModelPath(modelsDirPath);
         GlobalScope scope = new GlobalScope(mp, new CNNArchLanguage());
+        setModelPath(modelsDirPath);  
         generate(scope, rootModelName);
     }
 
@@ -120,6 +131,11 @@ public class CNNArch2Caffe2 implements CNNArchGenerator{
         }
 
         try{
+            String confPath = getModelPath() + "/data_paths.txt";
+            DataPathConfigParser newParserConfig = new DataPathConfigParser(confPath);
+            String dataPath = newParserConfig.getDataPath(rootModelName);
+            compilationUnit.get().getArchitecture().setDataPath(dataPath);
+            compilationUnit.get().getArchitecture().setComponentName(rootModelName);
             generateFiles(compilationUnit.get().getArchitecture());
         } catch (IOException e){
             Log.error(e.toString());

src/main/java/de/monticore/lang/monticar/cnnarch/caffe2generator/CNNArchTemplateController.java

@@ -89,6 +89,11 @@ public class CNNArchTemplateController {
         return getArchitecture().getEnclosingScope().getSpanningSymbol().get().getFullName().replaceAll("\\.","_");
     }
 
+    public String getDataPath(){
+        return getArchitecture().getDataPath();
+    }
+
+
     public List<String> getLayerInputs(ArchitectureElementSymbol layer){
         List<String> inputNames = new ArrayList<>();
 
@@ -125,6 +130,10 @@ public class CNNArchTemplateController {
         return list;
     }
 
+    public String getComponentName(){
+        return getArchitecture().getComponentName();
+    }
+
     public String getArchitectureLoss(){
         return this.loss;
     }

src/main/resources/templates/caffe2/CNNCreator.ftl

@@ -13,8 +13,8 @@ class ${tc.fileNameWithoutEnding}:
 
     module = None
     _current_dir_ = os.path.join('./')
-    _data_dir_    = os.path.join(_current_dir_, 'data', '${tc.fullArchitectureName}')
-    _model_dir_   = os.path.join(_current_dir_, 'model', '${tc.fullArchitectureName}')
+    _data_dir_    = os.path.join(_current_dir_, '${tc.dataPath}')
+    _model_dir_   = os.path.join(_current_dir_, 'model', '${tc.componentName}')
 
     _init_net_    = os.path.join(_model_dir_, 'init_net.pb')
     _predict_net_ = os.path.join(_model_dir_, 'predict_net.pb')

src/main/resources/templates/caffe2/CNNPredictor.ftl

@@ -18,8 +18,8 @@
 #include <iostream>
 #include <map>
 
-CAFFE2_DEFINE_string(init_net, "./model/${tc.fullArchitectureName}/init_net.pb", "The given path to the init protobuffer.");
-CAFFE2_DEFINE_string(predict_net, "./model/${tc.fullArchitectureName}/predict_net.pb", "The given path to the predict protobuffer.");
+CAFFE2_DEFINE_string(init_net_${tc.fileNameWithoutEnding}, "./model/${tc.componentName}/init_net.pb", "The given path to the init protobuffer.");
+CAFFE2_DEFINE_string(predict_net_${tc.fileNameWithoutEnding}, "./model/${tc.componentName}/predict_net.pb", "The given path to the predict protobuffer.");
 
 using namespace caffe2;
 
@@ -43,21 +43,21 @@ class ${tc.fileNameWithoutEnding}{
             char **a[1];
             caffe2::GlobalInit(&n, a);
 
-            if (!std::ifstream(FLAGS_init_net).good()) {
-                std::cerr << "\nNetwork loading failure, init_net file '" << FLAGS_init_net << "' does not exist." << std::endl;
+            if (!std::ifstream(FLAGS_init_net_${tc.fileNameWithoutEnding}).good()) {
+                std::cerr << "\nNetwork loading failure, init_net file '" << FLAGS_init_net_${tc.fileNameWithoutEnding} << "' does not exist." << std::endl;
                 exit(1);
             }
 
-            if (!std::ifstream(FLAGS_predict_net).good()) {
-                std::cerr << "\nNetwork loading failure, predict_net file '" << FLAGS_predict_net << "' does not exist." << std::endl;
+            if (!std::ifstream(FLAGS_predict_net_${tc.fileNameWithoutEnding}).good()) {
+                std::cerr << "\nNetwork loading failure, predict_net file '" << FLAGS_predict_net_${tc.fileNameWithoutEnding} << "' does not exist." << std::endl;
                 exit(1);
             }
 
             std::cout << "\nLoading network..." << std::endl;
 
             // Read protobuf
-            CAFFE_ENFORCE(ReadProtoFromFile(FLAGS_init_net, &initNet));
-            CAFFE_ENFORCE(ReadProtoFromFile(FLAGS_predict_net, &predictNet));
+            CAFFE_ENFORCE(ReadProtoFromFile(FLAGS_init_net_${tc.fileNameWithoutEnding}, &initNet));
+            CAFFE_ENFORCE(ReadProtoFromFile(FLAGS_predict_net_${tc.fileNameWithoutEnding}, &predictNet));
 
             // Set device type
             #ifdef USE_GPU

src/test/resources/architectures/data_paths.txt

+Alexnet data/Alexnet
+CifarClassifierNetwork data/CifarClassifierNetwork
+LeNet data/LeNet
+VGG16 data/VGG16
+MultipleOutputs data/MultipleOutputs
\ No newline at end of file

src/test/resources/invalid_tests/data_paths.txt

+Alexnet data/Alexnet
+CifarClassifierNetwork data/CifarClassifierNetwork
+LeNet data/LeNet
+VGG16 data/VGG16
+MultipleOutputs data/MultipleOutputs
\ No newline at end of file

src/test/resources/target_code/CNNCreator_Alexnet.py

@@ -12,7 +12,7 @@ class CNNCreator_Alexnet:
 
     module = None
     _current_dir_ = os.path.join('./')
-    _data_dir_    = os.path.join(_current_dir_, 'data', 'Alexnet')
+    _data_dir_    = os.path.join(_current_dir_, 'data/Alexnet')
     _model_dir_   = os.path.join(_current_dir_, 'model', 'Alexnet')
 
     _init_net_    = os.path.join(_model_dir_, 'init_net.pb')

src/test/resources/target_code/CNNCreator_CifarClassifierNetwork.py

@@ -12,7 +12,7 @@ class CNNCreator_CifarClassifierNetwork:
 
     module = None
     _current_dir_ = os.path.join('./')
-    _data_dir_    = os.path.join(_current_dir_, 'data', 'CifarClassifierNetwork')
+    _data_dir_    = os.path.join(_current_dir_, 'data/CifarClassifierNetwork')
     _model_dir_   = os.path.join(_current_dir_, 'model', 'CifarClassifierNetwork')
 
     _init_net_    = os.path.join(_model_dir_, 'init_net.pb')

src/test/resources/target_code/CNNCreator_LeNet.py

@@ -13,7 +13,7 @@ class CNNCreator_LeNet:
 
     module = None
     _current_dir_ = os.path.join('./')
-    _data_dir_    = os.path.join(_current_dir_, 'data', 'LeNet')
+    _data_dir_    = os.path.join(_current_dir_, 'data/LeNet')
     _model_dir_   = os.path.join(_current_dir_, 'model', 'LeNet')
 
     _init_net_    = os.path.join(_model_dir_, 'init_net.pb')

src/test/resources/target_code/CNNCreator_VGG16.py

@@ -13,7 +13,7 @@ class CNNCreator_VGG16:
 
     module = None
     _current_dir_ = os.path.join('./')
-    _data_dir_    = os.path.join(_current_dir_, 'data', 'VGG16')
+    _data_dir_    = os.path.join(_current_dir_, 'data/VGG16')
     _model_dir_   = os.path.join(_current_dir_, 'model', 'VGG16')
 
     _init_net_    = os.path.join(_model_dir_, 'init_net.pb')

src/test/resources/target_code/CNNPredictor_LeNet.h

@@ -18,8 +18,8 @@
 #include <iostream>
 #include <map>
 
-CAFFE2_DEFINE_string(init_net, "./model/LeNet/init_net.pb", "The given path to the init protobuffer.");
-CAFFE2_DEFINE_string(predict_net, "./model/LeNet/predict_net.pb", "The given path to the predict protobuffer.");
+CAFFE2_DEFINE_string(init_net_CNNPredictor_LeNet, "./model/LeNet/init_net.pb", "The given path to the init protobuffer.");
+CAFFE2_DEFINE_string(predict_net_CNNPredictor_LeNet, "./model/LeNet/predict_net.pb", "The given path to the predict protobuffer.");
 
 using namespace caffe2;
 
@@ -43,21 +43,21 @@ class CNNPredictor_LeNet{
             char **a[1];
             caffe2::GlobalInit(&n, a);
 
-            if (!std::ifstream(FLAGS_init_net).good()) {
-                std::cerr << "\nNetwork loading failure, init_net file '" << FLAGS_init_net << "' does not exist." << std::endl;
+            if (!std::ifstream(FLAGS_init_net_CNNPredictor_LeNet).good()) {
+                std::cerr << "\nNetwork loading failure, init_net file '" << FLAGS_init_net_CNNPredictor_LeNet << "' does not exist." << std::endl;
                 exit(1);
             }
 
-            if (!std::ifstream(FLAGS_predict_net).good()) {
-                std::cerr << "\nNetwork loading failure, predict_net file '" << FLAGS_predict_net << "' does not exist." << std::endl;
+            if (!std::ifstream(FLAGS_predict_net_CNNPredictor_LeNet).good()) {
+                std::cerr << "\nNetwork loading failure, predict_net file '" << FLAGS_predict_net_CNNPredictor_LeNet << "' does not exist." << std::endl;
                 exit(1);
             }
 
             std::cout << "\nLoading network..." << std::endl;
 
             // Read protobuf
-            CAFFE_ENFORCE(ReadProtoFromFile(FLAGS_init_net, &initNet));
-            CAFFE_ENFORCE(ReadProtoFromFile(FLAGS_predict_net, &predictNet));
+            CAFFE_ENFORCE(ReadProtoFromFile(FLAGS_init_net_CNNPredictor_LeNet, &initNet));
+            CAFFE_ENFORCE(ReadProtoFromFile(FLAGS_predict_net_CNNPredictor_LeNet, &predictNet));
 
             // Set device type
             #ifdef USE_GPU

src/test/resources/target_code/CNNPredictor_VGG16.h

@@ -18,8 +18,8 @@
 #include <iostream>
 #include <map>
 
-CAFFE2_DEFINE_string(init_net, "./model/VGG16/init_net.pb", "The given path to the init protobuffer.");
-CAFFE2_DEFINE_string(predict_net, "./model/VGG16/predict_net.pb", "The given path to the predict protobuffer.");
+CAFFE2_DEFINE_string(init_net_CNNPredictor_VGG16, "./model/VGG16/init_net.pb", "The given path to the init protobuffer.");
+CAFFE2_DEFINE_string(predict_net_CNNPredictor_VGG16, "./model/VGG16/predict_net.pb", "The given path to the predict protobuffer.");
 
 using namespace caffe2;
 
@@ -43,21 +43,21 @@ class CNNPredictor_VGG16{
             char **a[1];
             caffe2::GlobalInit(&n, a);
 
-            if (!std::ifstream(FLAGS_init_net).good()) {
-                std::cerr << "\nNetwork loading failure, init_net file '" << FLAGS_init_net << "' does not exist." << std::endl;
+            if (!std::ifstream(FLAGS_init_net_CNNPredictor_VGG16).good()) {
+                std::cerr << "\nNetwork loading failure, init_net file '" << FLAGS_init_net_CNNPredictor_VGG16 << "' does not exist." << std::endl;
                 exit(1);
             }
 
-            if (!std::ifstream(FLAGS_predict_net).good()) {
-                std::cerr << "\nNetwork loading failure, predict_net file '" << FLAGS_predict_net << "' does not exist." << std::endl;
+            if (!std::ifstream(FLAGS_predict_net_CNNPredictor_VGG16).good()) {
+                std::cerr << "\nNetwork loading failure, predict_net file '" << FLAGS_predict_net_CNNPredictor_VGG16 << "' does not exist." << std::endl;
                 exit(1);
             }
 
             std::cout << "\nLoading network..." << std::endl;
 
             // Read protobuf
-            CAFFE_ENFORCE(ReadProtoFromFile(FLAGS_init_net, &initNet));
-            CAFFE_ENFORCE(ReadProtoFromFile(FLAGS_predict_net, &predictNet));
+            CAFFE_ENFORCE(ReadProtoFromFile(FLAGS_init_net_CNNPredictor_VGG16, &initNet));
+            CAFFE_ENFORCE(ReadProtoFromFile(FLAGS_predict_net_CNNPredictor_VGG16, &predictNet));
 
             // Set device type
             #ifdef USE_GPU
@@ -114,4 +114,4 @@ class CNNPredictor_VGG16{
         }
 };
 
-#endif // CNNPREDICTOR_VGG16
\ No newline at end of file
+#endif // CNNPREDICTOR_VGG16

src/test/resources/valid_tests/data_paths.txt

+Alexnet data/Alexnet
+CifarClassifierNetwork data/CifarClassifierNetwork
+LeNet data/LeNet
+VGG16 data/VGG16
+MultipleOutputs data/MultipleOutputs
\ No newline at end of file