Skip to content

GitLab

Commit e86f9476 authored by Evgeny Kusmenko's avatar Evgeny Kusmenko
Browse files

Merge branch 'oneclick_nn_training' into 'master' 

Oneclick nn training

See merge request !29
parents 980d2c19 3cde5fb4
Pipeline #106739 passed with stages
in 3 minutes and 33 seconds
Hide whitespace changes
Inline Side-by-side
Showing with 78 additions and 38 deletions
pom.xml
View file @ e86f9476
......@@ -8,15 +8,15 @@
<groupId>de.monticore.lang.monticar</groupId>
<artifactId>cnnarch-caffe2-generator</artifactId>
<version>0.2.8-SNAPSHOT</version>
<version>0.2.8</version>
<!-- == PROJECT DEPENDENCIES ============================================= -->
<properties>
<!-- .. SE-Libraries .................................................. -->
<CNNArch.version>0.2.7</CNNArch.version>
<CNNTrain.version>0.2.6-SNAPSHOT</CNNTrain.version>
<CNNArch.version>0.2.8</CNNArch.version>
<CNNTrain.version>0.2.6</CNNTrain.version>
<embedded-montiarc-math-generator>0.1.2-SNAPSHOT</embedded-montiarc-math-generator>
<!-- .. Libraries .................................................. -->
......
settings.xml
View file @ e86f9476
......@@ -39,7 +39,7 @@
<id>se-nexus</id>
<username>cibuild</username>
<password>${env.cibuild}</password>
</server>
</server>
</servers>
<mirrors>
......@@ -47,7 +47,7 @@
<id>se-nexus</id>
<mirrorOf>external:*</mirrorOf>
<url>https://nexus.se.rwth-aachen.de/content/groups/public</url>
</mirror>
</mirror>
</mirrors>
<profiles>
......@@ -93,5 +93,5 @@
<activeProfiles>
<activeProfile>se-nexus</activeProfile>
</activeProfiles>
</settings>
\ No newline at end of file
</activeProfiles>
</settings>
src/main/java/de/monticore/lang/monticar/cnnarch/caffe2generator/CNNArch2Caffe2.java
View file @ e86f9476
......@@ -28,6 +28,7 @@ import de.monticore.lang.monticar.cnnarch._symboltable.ArchitectureElementSymbol
import de.monticore.lang.monticar.cnnarch._symboltable.CompositeElementSymbol;
import de.monticore.lang.monticar.cnnarch._symboltable.CNNArchCompilationUnitSymbol;
import de.monticore.lang.monticar.cnnarch._symboltable.CNNArchLanguage;
import de.monticore.lang.monticar.cnnarch.DataPathConfigParser;
import de.monticore.lang.monticar.generator.FileContent;
import de.monticore.lang.monticar.generator.cmake.CMakeConfig;
import de.monticore.lang.monticar.generator.cmake.CMakeFindModule;
......@@ -46,6 +47,7 @@ import java.util.List;
public class CNNArch2Caffe2 implements CNNArchGenerator{
private String generationTargetPath;
private String modelPath;
private boolean isSupportedLayer(ArchitectureElementSymbol element, LayerSupportChecker layerChecker){
List<ArchitectureElementSymbol> constructLayerElemList;
......@@ -76,6 +78,14 @@ public class CNNArch2Caffe2 implements CNNArchGenerator{
return true;
}
public String getModelPath(){
return modelPath;
}
public void setModelPath(Path modelPath){
this.modelPath = modelPath.toString();
}
private static void quitGeneration(){
Log.error("Code generation is aborted");
System.exit(1);
......@@ -104,6 +114,7 @@ public class CNNArch2Caffe2 implements CNNArchGenerator{
public void generate(Path modelsDirPath, String rootModelName){
final ModelPath mp = new ModelPath(modelsDirPath);
GlobalScope scope = new GlobalScope(mp, new CNNArchLanguage());
setModelPath(modelsDirPath);
generate(scope, rootModelName);
}
......@@ -120,6 +131,11 @@ public class CNNArch2Caffe2 implements CNNArchGenerator{
}
try{
String confPath = getModelPath() + "/data_paths.txt";
DataPathConfigParser newParserConfig = new DataPathConfigParser(confPath);
String dataPath = newParserConfig.getDataPath(rootModelName);
compilationUnit.get().getArchitecture().setDataPath(dataPath);
compilationUnit.get().getArchitecture().setComponentName(rootModelName);
generateFiles(compilationUnit.get().getArchitecture());
} catch (IOException e){
Log.error(e.toString());
......
src/main/java/de/monticore/lang/monticar/cnnarch/caffe2generator/CNNArchTemplateController.java
View file @ e86f9476
......@@ -89,6 +89,11 @@ public class CNNArchTemplateController {
return getArchitecture().getEnclosingScope().getSpanningSymbol().get().getFullName().replaceAll("\\.","_");
}
public String getDataPath(){
return getArchitecture().getDataPath();
}
public List<String> getLayerInputs(ArchitectureElementSymbol layer){
List<String> inputNames = new ArrayList<>();
......@@ -125,6 +130,10 @@ public class CNNArchTemplateController {
return list;
}
public String getComponentName(){
return getArchitecture().getComponentName();
}
public String getArchitectureLoss(){
return this.loss;
}
......
src/main/resources/templates/caffe2/CNNCreator.ftl
View file @ e86f9476
......@@ -13,8 +13,8 @@ class ${tc.fileNameWithoutEnding}:
module = None
_current_dir_ = os.path.join('./')
_data_dir_ = os.path.join(_current_dir_, 'data', '${tc.fullArchitectureName}')
_model_dir_ = os.path.join(_current_dir_, 'model', '${tc.fullArchitectureName}')
_data_dir_ = os.path.join(_current_dir_, '${tc.dataPath}')
_model_dir_ = os.path.join(_current_dir_, 'model', '${tc.componentName}')
_init_net_ = os.path.join(_model_dir_, 'init_net.pb')
_predict_net_ = os.path.join(_model_dir_, 'predict_net.pb')
......
src/main/resources/templates/caffe2/CNNPredictor.ftl
View file @ e86f9476
......@@ -18,8 +18,8 @@
#include <iostream>
#include <map>
CAFFE2_DEFINE_string(init_net, "./model/${tc.fullArchitectureName}/init_net.pb", "The given path to the init protobuffer.");
CAFFE2_DEFINE_string(predict_net, "./model/${tc.fullArchitectureName}/predict_net.pb", "The given path to the predict protobuffer.");
CAFFE2_DEFINE_string(init_net_${tc.fileNameWithoutEnding}, "./model/${tc.componentName}/init_net.pb", "The given path to the init protobuffer.");
CAFFE2_DEFINE_string(predict_net_${tc.fileNameWithoutEnding}, "./model/${tc.componentName}/predict_net.pb", "The given path to the predict protobuffer.");
using namespace caffe2;
......@@ -43,21 +43,21 @@ class ${tc.fileNameWithoutEnding}{
char **a[1];
caffe2::GlobalInit(&n, a);
if (!std::ifstream(FLAGS_init_net).good()) {
std::cerr << "\nNetwork loading failure, init_net file '" << FLAGS_init_net << "' does not exist." << std::endl;
if (!std::ifstream(FLAGS_init_net_${tc.fileNameWithoutEnding}).good()) {
std::cerr << "\nNetwork loading failure, init_net file '" << FLAGS_init_net_${tc.fileNameWithoutEnding} << "' does not exist." << std::endl;
exit(1);
}
if (!std::ifstream(FLAGS_predict_net).good()) {
std::cerr << "\nNetwork loading failure, predict_net file '" << FLAGS_predict_net << "' does not exist." << std::endl;
if (!std::ifstream(FLAGS_predict_net_${tc.fileNameWithoutEnding}).good()) {
std::cerr << "\nNetwork loading failure, predict_net file '" << FLAGS_predict_net_${tc.fileNameWithoutEnding} << "' does not exist." << std::endl;
exit(1);
}
std::cout << "\nLoading network..." << std::endl;
// Read protobuf
CAFFE_ENFORCE(ReadProtoFromFile(FLAGS_init_net, &initNet));
CAFFE_ENFORCE(ReadProtoFromFile(FLAGS_predict_net, &predictNet));
CAFFE_ENFORCE(ReadProtoFromFile(FLAGS_init_net_${tc.fileNameWithoutEnding}, &initNet));
CAFFE_ENFORCE(ReadProtoFromFile(FLAGS_predict_net_${tc.fileNameWithoutEnding}, &predictNet));
// Set device type
#ifdef USE_GPU
......
src/test/resources/architectures/data_paths.txt 0 → 100644
View file @ e86f9476
Alexnet data/Alexnet
CifarClassifierNetwork data/CifarClassifierNetwork
LeNet data/LeNet
VGG16 data/VGG16
MultipleOutputs data/MultipleOutputs
\ No newline at end of file
src/test/resources/invalid_tests/data_paths.txt 0 → 100644
View file @ e86f9476
Alexnet data/Alexnet
CifarClassifierNetwork data/CifarClassifierNetwork
LeNet data/LeNet
VGG16 data/VGG16
MultipleOutputs data/MultipleOutputs
\ No newline at end of file
src/test/resources/target_code/CNNCreator_Alexnet.py
View file @ e86f9476
......@@ -12,7 +12,7 @@ class CNNCreator_Alexnet:
module = None
_current_dir_ = os.path.join('./')
_data_dir_ = os.path.join(_current_dir_, 'data', 'Alexnet')
_data_dir_ = os.path.join(_current_dir_, 'data/Alexnet')
_model_dir_ = os.path.join(_current_dir_, 'model', 'Alexnet')
_init_net_ = os.path.join(_model_dir_, 'init_net.pb')
......
src/test/resources/target_code/CNNCreator_CifarClassifierNetwork.py
View file @ e86f9476
......@@ -12,7 +12,7 @@ class CNNCreator_CifarClassifierNetwork:
module = None
_current_dir_ = os.path.join('./')
_data_dir_ = os.path.join(_current_dir_, 'data', 'CifarClassifierNetwork')
_data_dir_ = os.path.join(_current_dir_, 'data/CifarClassifierNetwork')
_model_dir_ = os.path.join(_current_dir_, 'model', 'CifarClassifierNetwork')
_init_net_ = os.path.join(_model_dir_, 'init_net.pb')
......
src/test/resources/target_code/CNNCreator_LeNet.py
View file @ e86f9476
......@@ -13,7 +13,7 @@ class CNNCreator_LeNet:
module = None
_current_dir_ = os.path.join('./')
_data_dir_ = os.path.join(_current_dir_, 'data', 'LeNet')
_data_dir_ = os.path.join(_current_dir_, 'data/LeNet')
_model_dir_ = os.path.join(_current_dir_, 'model', 'LeNet')
_init_net_ = os.path.join(_model_dir_, 'init_net.pb')
......
src/test/resources/target_code/CNNCreator_VGG16.py
View file @ e86f9476
......@@ -13,7 +13,7 @@ class CNNCreator_VGG16:
module = None
_current_dir_ = os.path.join('./')
_data_dir_ = os.path.join(_current_dir_, 'data', 'VGG16')
_data_dir_ = os.path.join(_current_dir_, 'data/VGG16')
_model_dir_ = os.path.join(_current_dir_, 'model', 'VGG16')
_init_net_ = os.path.join(_model_dir_, 'init_net.pb')
......
src/test/resources/target_code/CNNPredictor_LeNet.h
View file @ e86f9476
......@@ -18,8 +18,8 @@
#include <iostream>
#include <map>
CAFFE2_DEFINE_string(init_net, "./model/LeNet/init_net.pb", "The given path to the init protobuffer.");
CAFFE2_DEFINE_string(predict_net, "./model/LeNet/predict_net.pb", "The given path to the predict protobuffer.");
CAFFE2_DEFINE_string(init_net_CNNPredictor_LeNet, "./model/LeNet/init_net.pb", "The given path to the init protobuffer.");
CAFFE2_DEFINE_string(predict_net_CNNPredictor_LeNet, "./model/LeNet/predict_net.pb", "The given path to the predict protobuffer.");
using namespace caffe2;
......@@ -43,21 +43,21 @@ class CNNPredictor_LeNet{
char **a[1];
caffe2::GlobalInit(&n, a);
if (!std::ifstream(FLAGS_init_net).good()) {
std::cerr << "\nNetwork loading failure, init_net file '" << FLAGS_init_net << "' does not exist." << std::endl;
if (!std::ifstream(FLAGS_init_net_CNNPredictor_LeNet).good()) {
std::cerr << "\nNetwork loading failure, init_net file '" << FLAGS_init_net_CNNPredictor_LeNet << "' does not exist." << std::endl;
exit(1);
}
if (!std::ifstream(FLAGS_predict_net).good()) {
std::cerr << "\nNetwork loading failure, predict_net file '" << FLAGS_predict_net << "' does not exist." << std::endl;
if (!std::ifstream(FLAGS_predict_net_CNNPredictor_LeNet).good()) {
std::cerr << "\nNetwork loading failure, predict_net file '" << FLAGS_predict_net_CNNPredictor_LeNet << "' does not exist." << std::endl;
exit(1);
}
std::cout << "\nLoading network..." << std::endl;
// Read protobuf
CAFFE_ENFORCE(ReadProtoFromFile(FLAGS_init_net, &initNet));
CAFFE_ENFORCE(ReadProtoFromFile(FLAGS_predict_net, &predictNet));
CAFFE_ENFORCE(ReadProtoFromFile(FLAGS_init_net_CNNPredictor_LeNet, &initNet));
CAFFE_ENFORCE(ReadProtoFromFile(FLAGS_predict_net_CNNPredictor_LeNet, &predictNet));
// Set device type
#ifdef USE_GPU
......
src/test/resources/target_code/CNNPredictor_VGG16.h
View file @ e86f9476
......@@ -18,8 +18,8 @@
#include <iostream>
#include <map>
CAFFE2_DEFINE_string(init_net, "./model/VGG16/init_net.pb", "The given path to the init protobuffer.");
CAFFE2_DEFINE_string(predict_net, "./model/VGG16/predict_net.pb", "The given path to the predict protobuffer.");
CAFFE2_DEFINE_string(init_net_CNNPredictor_VGG16, "./model/VGG16/init_net.pb", "The given path to the init protobuffer.");
CAFFE2_DEFINE_string(predict_net_CNNPredictor_VGG16, "./model/VGG16/predict_net.pb", "The given path to the predict protobuffer.");
using namespace caffe2;
......@@ -43,21 +43,21 @@ class CNNPredictor_VGG16{
char **a[1];
caffe2::GlobalInit(&n, a);
if (!std::ifstream(FLAGS_init_net).good()) {
std::cerr << "\nNetwork loading failure, init_net file '" << FLAGS_init_net << "' does not exist." << std::endl;
if (!std::ifstream(FLAGS_init_net_CNNPredictor_VGG16).good()) {
std::cerr << "\nNetwork loading failure, init_net file '" << FLAGS_init_net_CNNPredictor_VGG16 << "' does not exist." << std::endl;
exit(1);
}
if (!std::ifstream(FLAGS_predict_net).good()) {
std::cerr << "\nNetwork loading failure, predict_net file '" << FLAGS_predict_net << "' does not exist." << std::endl;
if (!std::ifstream(FLAGS_predict_net_CNNPredictor_VGG16).good()) {
std::cerr << "\nNetwork loading failure, predict_net file '" << FLAGS_predict_net_CNNPredictor_VGG16 << "' does not exist." << std::endl;
exit(1);
}
std::cout << "\nLoading network..." << std::endl;
// Read protobuf
CAFFE_ENFORCE(ReadProtoFromFile(FLAGS_init_net, &initNet));
CAFFE_ENFORCE(ReadProtoFromFile(FLAGS_predict_net, &predictNet));
CAFFE_ENFORCE(ReadProtoFromFile(FLAGS_init_net_CNNPredictor_VGG16, &initNet));
CAFFE_ENFORCE(ReadProtoFromFile(FLAGS_predict_net_CNNPredictor_VGG16, &predictNet));
// Set device type
#ifdef USE_GPU
......@@ -114,4 +114,4 @@ class CNNPredictor_VGG16{
}
};
#endif // CNNPREDICTOR_VGG16
\ No newline at end of file
#endif // CNNPREDICTOR_VGG16
src/test/resources/valid_tests/data_paths.txt 0 → 100644
View file @ e86f9476
Alexnet data/Alexnet
CifarClassifierNetwork data/CifarClassifierNetwork
LeNet data/LeNet
VGG16 data/VGG16
MultipleOutputs data/MultipleOutputs
\ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment