Commit 8f0fa3d0 authored by Carlos Alfredo Yeverino Rodriguez's avatar Carlos Alfredo Yeverino Rodriguez
Browse files

Merge branch 'master' into layer-templates-caffe2

parents 9bacfee9 058f8ab3
...@@ -27,13 +27,15 @@ masterJobLinux: ...@@ -27,13 +27,15 @@ masterJobLinux:
stage: linux stage: linux
image: maven:3-jdk-8 image: maven:3-jdk-8
script: script:
- mvn -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -B clean deploy --settings settings.xml - mvn -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -B clean install deploy --settings settings.xml
- cat target/site/jacoco/index.html
- mvn package sonar:sonar -s settings.xml
only: only:
- master - master
masterJobWindows: masterJobWindows:
stage: windows stage: windows
script: script:
- mvn -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -B clean install --settings settings.xml - mvn -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -B clean install --settings settings.xml
tags: tags:
- Windows10 - Windows10
...@@ -43,5 +45,6 @@ BranchJobLinux: ...@@ -43,5 +45,6 @@ BranchJobLinux:
image: maven:3-jdk-8 image: maven:3-jdk-8
script: script:
- mvn -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -B clean install --settings settings.xml - mvn -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -B clean install --settings settings.xml
- cat target/site/jacoco/index.html
except: except:
- master - master
This generator generates caffe2 networks from the cnnarch language. ![pipeline](https://git.rwth-aachen.de/monticore/EmbeddedMontiArc/generators/CNNArch2Caffe2/badges/master/build.svg)
\ No newline at end of file ![coverage](https://git.rwth-aachen.de/monticore/EmbeddedMontiArc/generators/CNNArch2Caffe2/badges/master/coverage.svg)
# CNNArch2Caffe2
This generator generates caffe2 networks from the cnnarch language.
## Caffe2 Installation Guide
This is a guide about how to install Caffe2 on Ubuntu and Windows 10.
1. [Ubuntu](#ubuntu)
2. [Windows 10](#windows-10)
### Ubuntu
This guide is based on the installation guide from the Caffe2 website: https://caffe2.ai/docs/getting-started.html?platform=ubuntu&configuration=compile
- **For GPU Support :** Install CUDA 8.0 and cuDNN directly from NVIDIA website.
Install CUDA 8.0 with the option "deb (local)". Install cuDNN with the option "Library for Linux" and download cuDNN 7.0.5 since version 5.1 does not work with CUDA 8.0.
Do the following in the extracted folder cuDNN 7.0.5:
```
sudo cp -P include/cudnn.h /usr/local/cuda-8.0/include
sudo cp -P lib64/libcudnn* /usr/local/cuda-8.0/lib64
```
- Install Dependencies:
```
sudo apt-get update
sudo apt-get install -y --no-install-recommends \
build-essential \
cmake \
git \
libgoogle-glog-dev \
libgtest-dev \
libiomp-dev \
libleveldb-dev \
liblmdb-dev \
libopencv-dev \
libopenmpi-dev \
libsnappy-dev \
libprotobuf-dev \
openmpi-bin \
openmpi-doc \
protobuf-compiler \
python-dev \
python-pip
pip install --user \
future \
numpy \
protobuf
```
```
#for Ubuntu 14.04
sudo apt-get install -y --no-install-recommends libgflags2
#for Ubuntu 16.04
sudo apt-get install -y --no-install-recommends libgflags-dev
```
- Set PYTHONPATH and LD_LIBRARY_PATH before compiling Caffe2. In terminal (anywhere), do the following:
```
sudo gedit ~/.bashrc
```
Add the following at the end of the file:
```
#caffe2
#echo $PYTHONPATH
export PYTHONPATH = /usr/local
export PYTHONPATH = $PYTHONPATH:YourRepositoryPath/pytorch/build
export PYTHONPATH = $PYTHONPATH:/usr/bin/python
#echo $LD_LIBRARY_PATH
export LD_LIBRARY_PATH = /usr/local/lib
```
Save the file and close it. Then, in terminal do the following:
```
source ~/.bashrc
```
Now, in terminal execute
```
echo $PYTHONPATH
echo $LD_LIBRARY_PATH
```
And review that there should not be double ":" in the paths.
- Clone Pytorch which now contains Caffe2 in your desired YourRepositoryPath and compile it:
```
git clone --recursive https://github.com/pytorch/pytorch.git && cd pythorch
git submodule update --init
mkdir build && cd build
cmake..
```
Review the output from cmake configuration. CUDA value should be CUDA 8.0 and cuDNN value should be 7.0.5.
```
sudo make install
```
Review that progress should achieve 100% and there should be no error messages.
- Test Caffe2 installation:
In terminal (anywhere), do the following:
```
cd ~ && python -c 'from caffe2.python import core'
2>/dev/null && echo "Success" || echo "Failure"
```
Review the output. Output should be Success.
### Windows 10
This guide is based on the installation guide from the Caffe2 website: https://caffe2.ai/docs/getting-started.html?platform=windows&configuration=compile
- Add the directories C:\Python27 and C:\Python27\Scripts to the system variable _Path_.
- Install python dependencies. Run the command line as administrator and navigate to path C:\Python27\Scripts. Execute the following commands:
```
pip install future
pip install hypothesis (if waring appears, then add C:\phyton27\Scripts to _Path_)
pip install numpy
pip install protobuf
pip install six
```
Optional packages:
```
pip install flask
pip install glog
pip install graphviz
pip install jupyter
pip install matplotlib
pip install pydot python-nvd3
pip install pyyaml
pip install requests
pip install scikit-image
pip install scipy
pip install setuptools
pip install tornado
```
If there is an error installing optional python packages with “pip install …”, e.g., for graphviz or matplotlib, then execute the following in the same root:
```
pip install - - upgrade setuptools.
```
- Install CMake (Add CMake bin to _Path_)
- Install Git from https://git-scm.com/downloads
- Clone Pytorch which now contains Caffe2 in your desired YourRepositoryPath:
```
git clone --recursive https://github.com/pytorch/pytorch.git
```
- Add the system variable CAFFE2_ROOT with the directory YourRepositoryPath\pytorch. Then, add the directory %CAFFE2_ROOT%\bin to _Path_.
Afterwards, modify the file \scripts\build_windows.bat to set BUILD_PYTHON to ON instead of OFF.
Open a Developer Command Prompt for VS 2017 (from Visual Studio 2017) and navigate to \scripts and execute build_windows.bat. Here the compilation will take place.
```
build_windows.bat
```
- **For GPU Support :** Before executing build_windows.bat, set the following:
```
Set CMAKE_GENERATOR=(quote)Visual Studio 14 2015 Win64(quote)
Set USE_CUDA=ON
Set TORCH_CUDA_ARCH_LIST=6.1  for GeForce GTX 1050 architecture
```
Then execute build.windows.bat
- Once Caffe2 is successfully compiled, add the system variable PYTHONPATH with the following directories in order to fix the possible error “ImportError: No module named Caffe2.python”:
C:\Python27\Lib;
C:\Phyton27\DLLs;
C:\Phyton27\Lib\lib-tk
C:\Users\Carlos\Documents\git\pytorch\build
...@@ -8,15 +8,16 @@ ...@@ -8,15 +8,16 @@
<groupId>de.monticore.lang.monticar</groupId> <groupId>de.monticore.lang.monticar</groupId>
<artifactId>cnnarch-caffe2-generator</artifactId> <artifactId>cnnarch-caffe2-generator</artifactId>
<version>0.2.2-SNAPSHOT</version> <version>0.2.5-SNAPSHOT</version>
<!-- == PROJECT DEPENDENCIES ============================================= --> <!-- == PROJECT DEPENDENCIES ============================================= -->
<properties> <properties>
<!-- .. SE-Libraries .................................................. --> <!-- .. SE-Libraries .................................................. -->
<CNNArch.version>0.2.3-SNAPSHOT</CNNArch.version> <CNNArch.version>0.2.6-SNAPSHOT</CNNArch.version>
<CNNTrain.version>0.2.4-SNAPSHOT</CNNTrain.version> <CNNTrain.version>0.2.5-SNAPSHOT</CNNTrain.version>
<embedded-montiarc-math-generator>0.0.25-SNAPSHOT</embedded-montiarc-math-generator>
<!-- .. Libraries .................................................. --> <!-- .. Libraries .................................................. -->
<guava.version>18.0</guava.version> <guava.version>18.0</guava.version>
...@@ -29,6 +30,7 @@ ...@@ -29,6 +30,7 @@
<compiler.plugin>3.3</compiler.plugin> <compiler.plugin>3.3</compiler.plugin>
<source.plugin>2.4</source.plugin> <source.plugin>2.4</source.plugin>
<shade.plugin>2.4.3</shade.plugin> <shade.plugin>2.4.3</shade.plugin>
<jacoco.plugin>0.8.1</jacoco.plugin>
<!-- Classifiers --> <!-- Classifiers -->
<grammars.classifier>grammars</grammars.classifier> <grammars.classifier>grammars</grammars.classifier>
...@@ -85,6 +87,12 @@ ...@@ -85,6 +87,12 @@
<scope>provided</scope> <scope>provided</scope>
</dependency> </dependency>
<dependency>
<groupId>de.monticore.lang.monticar</groupId>
<artifactId>embedded-montiarc-math-generator</artifactId>
<version>${embedded-montiarc-math-generator}</version>
</dependency>
<!-- .. Test Libraries ............................................... --> <!-- .. Test Libraries ............................................... -->
<dependency> <dependency>
...@@ -117,6 +125,28 @@ ...@@ -117,6 +125,28 @@
<artifactId>maven-deploy-plugin</artifactId> <artifactId>maven-deploy-plugin</artifactId>
<version>2.8.1</version> <version>2.8.1</version>
</plugin> </plugin>
<!-- Test coverage -->
<plugin>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
<version>${jacoco.plugin}</version>
<executions>
<execution>
<id>pre-unit-test</id>
<goals>
<goal>prepare-agent</goal>
</goals>
</execution>
<execution>
<id>post-unit-test</id>
<phase>test</phase>
<goals>
<goal>report</goal>
</goals>
</execution>
</executions>
</plugin>
<!-- Other Configuration --> <!-- Other Configuration -->
<plugin> <plugin>
......
...@@ -51,6 +51,24 @@ ...@@ -51,6 +51,24 @@
</mirrors> </mirrors>
<profiles> <profiles>
<profile>
<id>sonar</id>
<activation>
<activeByDefault>true</activeByDefault>
</activation>
<properties>
<!-- Optional URL to server. Default value is http://localhost:9000 -->
<sonar.host.url>
https://metric.se.rwth-aachen.de
</sonar.host.url>
<sonar.login>
jenkins
</sonar.login>
<sonar.password>
${env.sonar}
</sonar.password>
</properties>
</profile>
<profile> <profile>
<id>se-nexus</id> <id>se-nexus</id>
......
...@@ -26,16 +26,19 @@ import de.monticore.lang.monticar.cnnarch._cocos.CNNArchCocos; ...@@ -26,16 +26,19 @@ import de.monticore.lang.monticar.cnnarch._cocos.CNNArchCocos;
import de.monticore.lang.monticar.cnnarch._symboltable.ArchitectureSymbol; import de.monticore.lang.monticar.cnnarch._symboltable.ArchitectureSymbol;
import de.monticore.lang.monticar.cnnarch._symboltable.CNNArchCompilationUnitSymbol; import de.monticore.lang.monticar.cnnarch._symboltable.CNNArchCompilationUnitSymbol;
import de.monticore.lang.monticar.cnnarch._symboltable.CNNArchLanguage; import de.monticore.lang.monticar.cnnarch._symboltable.CNNArchLanguage;
import de.monticore.lang.monticar.cnntrain._symboltable.ConfigurationSymbol; import de.monticore.lang.monticar.generator.FileContent;
import de.monticore.lang.monticar.generator.cmake.CMakeConfig;
import de.monticore.lang.monticar.generator.cmake.CMakeFindModule;
import de.monticore.lang.monticar.generator.cpp.GeneratorCPP;
import de.monticore.symboltable.GlobalScope; import de.monticore.symboltable.GlobalScope;
import de.monticore.symboltable.Scope; import de.monticore.symboltable.Scope;
import de.se_rwth.commons.logging.Log; import de.se_rwth.commons.logging.Log;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException; import java.io.IOException;
import java.nio.file.Path; import java.nio.file.Path;
import java.util.*; import java.util.HashMap;
import java.util.Map;
import java.util.Optional;
public class CNNArch2Caffe2 implements CNNArchGenerator{ public class CNNArch2Caffe2 implements CNNArchGenerator{
...@@ -45,6 +48,11 @@ public class CNNArch2Caffe2 implements CNNArchGenerator{ ...@@ -45,6 +48,11 @@ public class CNNArch2Caffe2 implements CNNArchGenerator{
setGenerationTargetPath("./target/generated-sources-cnnarch/"); setGenerationTargetPath("./target/generated-sources-cnnarch/");
} }
@Override
public boolean isCMakeRequired() {
return true;
}
public String getGenerationTargetPath() { public String getGenerationTargetPath() {
if (generationTargetPath.charAt(generationTargetPath.length() - 1) != '/') { if (generationTargetPath.charAt(generationTargetPath.length() - 1) != '/') {
this.generationTargetPath = generationTargetPath + "/"; this.generationTargetPath = generationTargetPath + "/";
...@@ -79,24 +87,6 @@ public class CNNArch2Caffe2 implements CNNArchGenerator{ ...@@ -79,24 +87,6 @@ public class CNNArch2Caffe2 implements CNNArchGenerator{
} }
} }
@Override
public Map<String, String> generateTrainer(List<ConfigurationSymbol> configurations, List<String> instanceNames, String mainComponentName) {
int numberOfNetworks = configurations.size();
if (configurations.size() != instanceNames.size()){
throw new IllegalStateException(
"The number of configurations and the number of instances for generation of the CNNTrainer is not equal. " +
"This should have been checked previously.");
}
List<ConfigurationData> configDataList = new ArrayList<>();
for(int i = 0; i < numberOfNetworks; i++){
configDataList.add(new ConfigurationData(configurations.get(i), instanceNames.get(i)));
}
Map<String, Object> ftlContext = Collections.singletonMap("configurations", configDataList);
return Collections.singletonMap(
"CNNTrainer_" + mainComponentName + ".py",
TemplateConfiguration.processTemplate(ftlContext, "CNNTrainer.ftl"));
}
//check cocos with CNNArchCocos.checkAll(architecture) before calling this method. //check cocos with CNNArchCocos.checkAll(architecture) before calling this method.
public Map<String, String> generateStrings(ArchitectureSymbol architecture){ public Map<String, String> generateStrings(ArchitectureSymbol architecture){
Map<String, String> fileContentMap = new HashMap<>(); Map<String, String> fileContentMap = new HashMap<>();
...@@ -123,40 +113,58 @@ public class CNNArch2Caffe2 implements CNNArchGenerator{ ...@@ -123,40 +113,58 @@ public class CNNArch2Caffe2 implements CNNArchGenerator{
private void checkValidGeneration(ArchitectureSymbol architecture){ private void checkValidGeneration(ArchitectureSymbol architecture){
if (architecture.getInputs().size() > 1){ if (architecture.getInputs().size() > 1){
Log.error("This cnn architecture has multiple inputs, " + Log.error("This cnn architecture has multiple inputs, " +
"which is currently not supported by the generator. " "which is currently not supported by the caffe2generator. "
, architecture.getSourcePosition()); , architecture.getSourcePosition());
} }
if (architecture.getOutputs().size() > 1){ if (architecture.getOutputs().size() > 1){
Log.error("This cnn architecture has multiple outputs, " + Log.error("This cnn architecture has multiple outputs, " +
"which is currently not supported by the generator. " "which is currently not supported by the caffe2generator. "
, architecture.getSourcePosition()); , architecture.getSourcePosition());
} }
if (architecture.getOutputs().get(0).getDefinition().getType().getWidth() != 1 || if (architecture.getOutputs().get(0).getDefinition().getType().getWidth() != 1 ||
architecture.getOutputs().get(0).getDefinition().getType().getHeight() != 1){ architecture.getOutputs().get(0).getDefinition().getType().getHeight() != 1){
Log.error("This cnn architecture has a multi-dimensional output, " + Log.error("This cnn architecture has a multi-dimensional output, " +
"which is currently not supported by the generator." "which is currently not supported by the caffe2generator."
, architecture.getSourcePosition()); , architecture.getSourcePosition());
} }
} }
//check cocos with CNNArchCocos.checkAll(architecture) before calling this method. //check cocos with CNNArchCocos.checkAll(architecture) before calling this method.
public void generateFiles(ArchitectureSymbol architecture) throws IOException{ public void generateFiles(ArchitectureSymbol architecture) throws IOException{
CNNArchTemplateController archTc = new CNNArchTemplateController(architecture);
Map<String, String> fileContentMap = generateStrings(architecture); Map<String, String> fileContentMap = generateStrings(architecture);
generateFromFilecontentsMap(fileContentMap);
}
public void generateCMake(String rootModelName) {
Map<String, String> fileContentMap = generateCMakeContent(rootModelName);
try {
generateFromFilecontentsMap(fileContentMap);
} catch (IOException e) {
e.printStackTrace();
}
}
private void generateFromFilecontentsMap(Map<String, String> fileContentMap) throws IOException {
GeneratorCPP genCPP = new GeneratorCPP();
genCPP.setGenerationTargetPath(getGenerationTargetPath());
for (String fileName : fileContentMap.keySet()){ for (String fileName : fileContentMap.keySet()){
File f = new File(getGenerationTargetPath() + fileName); genCPP.generateFile(new FileContent(fileContentMap.get(fileName), fileName));
Log.info(f.getName(), "FileCreation:");
if (!f.exists()) {
f.getParentFile().mkdirs();
if (!f.createNewFile()) {
Log.error("File could not be created");
}
}
FileWriter writer = new FileWriter(f);
writer.write(fileContentMap.get(fileName));
writer.close();
} }
} }
public Map<String, String> generateCMakeContent(String rootModelName) {
// model name should start with a lower case letter. If it is a component, replace dot . by _
rootModelName = rootModelName.replace('.', '_').replace('[', '_').replace(']', '_');
rootModelName = rootModelName.substring(0, 1).toLowerCase() + rootModelName.substring(1);
CMakeConfig cMakeConfig = new CMakeConfig(rootModelName);
cMakeConfig.addModuleDependency(new CMakeFindModule("Armadillo", true));
cMakeConfig.addCMakeCommand("set(LIBS ${LIBS} mxnet)");
Map<String,String> fileContentMap = new HashMap<>();
for (FileContent fileContent : cMakeConfig.generateCMakeFiles()){
fileContentMap.put(fileContent.getFileName(), fileContent.getFileContent());
}
return fileContentMap;
}
} }
package de.monticore.lang.monticar.cnnarch.caffe2generator;
import de.monticore.io.paths.ModelPath;
import de.monticore.lang.monticar.cnntrain.CNNTrainGenerator;
import de.monticore.lang.monticar.cnntrain._cocos.CNNTrainCocos;
import de.monticore.lang.monticar.cnntrain._symboltable.CNNTrainCompilationUnitSymbol;
import de.monticore.lang.monticar.cnntrain._symboltable.CNNTrainLanguage;
import de.monticore.lang.monticar.cnntrain._symboltable.ConfigurationSymbol;
import de.monticore.lang.monticar.generator.FileContent;
import de.monticore.lang.monticar.generator.cpp.GeneratorCPP;
import de.monticore.symboltable.GlobalScope;
import de.se_rwth.commons.logging.Log;
import java.io.IOException;
import java.nio.file.Path;
import java.util.*;
public class CNNTrain2Caffe2 implements CNNTrainGenerator {
private String generationTargetPath;
private String instanceName;
public CNNTrain2Caffe2() {
setGenerationTargetPath("./target/generated-sources-cnnarch/");
}
public String getInstanceName() {
String parsedInstanceName = this.instanceName.replace('.', '_').replace('[', '_').replace(']', '_');
parsedInstanceName = parsedInstanceName.substring(0, 1).toLowerCase() + parsedInstanceName.substring(1);
return parsedInstanceName;
}
public void setInstanceName(String instanceName) {
this.instanceName = instanceName;
}
public String getGenerationTargetPath() {
if (generationTargetPath.charAt(generationTargetPath.length() - 1) != '/') {
this.generationTargetPath = generationTargetPath + "/";
}
return generationTargetPath;
}
public void setGenerationTargetPath(String generationTargetPath) {
this.generationTargetPath = generationTargetPath;
}
public ConfigurationSymbol getConfigurationSymbol(Path modelsDirPath, String rootModelName) {
final ModelPath mp = new ModelPath(modelsDirPath);
GlobalScope scope = new GlobalScope(mp, new CNNTrainLanguage());
Optional<CNNTrainCompilationUnitSymbol> compilationUnit = scope.resolve(rootModelName, CNNTrainCompilationUnitSymbol.KIND);
if (!compilationUnit.isPresent()) {
Log.error("could not resolve training configuration " + rootModelName);
System.exit(1);
}
setInstanceName(compilationUnit.get().getFullName());
CNNTrainCocos.checkAll(compilationUnit.get());
return compilationUnit.get().getConfiguration();
}
public void generate(Path modelsDirPath, String rootModelName) {
ConfigurationSymbol configuration = getConfigurationSymbol(modelsDirPath, rootModelName);
Map<String, String> fileContents = generateStrings(configuration);
GeneratorCPP genCPP = new GeneratorCPP();
genCPP.setGenerationTargetPath(getGenerationTargetPath());
try {
for (String fileName : fileContents.keySet()){
genCPP.generateFile(new FileContent(fileContents.get(fileName), fileName));
}
} catch (IOException e) {
e.printStackTrace();