...
 
Commits (189)
#
#
# ******************************************************************************
# MontiCAR Modeling Family, www.se-rwth.de
# Copyright (c) 2017, Software Engineering Group at RWTH Aachen,
# All rights reserved.
#
# This project is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 3.0 of the License, or (at your option) any later version.
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this project. If not, see <http://www.gnu.org/licenses/>.
# *******************************************************************************
#
# (c) https://github.com/MontiCore/monticore
# Java Maven CircleCI 2.0 configuration file
#
......
......@@ -5,6 +5,7 @@ nppBackup
.classpath
.idea
.git
.vscode
*.iml
train.log
model
#
#
# ******************************************************************************
# MontiCAR Modeling Family, www.se-rwth.de
# Copyright (c) 2017, Software Engineering Group at RWTH Aachen,
# All rights reserved.
#
# This project is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 3.0 of the License, or (at your option) any later version.
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this project. If not, see <http://www.gnu.org/licenses/>.
# *******************************************************************************
#
# (c) https://github.com/MontiCore/monticore
stages:
- windows
- linux
- deploy
masterJobLinux:
git masterJobLinux:
stage: deploy
image: maven:3-jdk-8
script:
......@@ -34,9 +16,10 @@ masterJobLinux:
only:
- master
integrationMXNetJobLinux:
stage: linux
image: registry.git.rwth-aachen.de/monticore/embeddedmontiarc/generators/emadl2cpp/integrationtests/mxnet:v0.0.3
image: registry.git.rwth-aachen.de/monticore/embeddedmontiarc/generators/emadl2cpp/integrationtests/mxnet:v0.0.4
script:
- mvn -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -B clean install --settings settings.xml -Dtest=IntegrationMXNetTest
......@@ -47,17 +30,31 @@ integrationCaffe2JobLinux:
script:
- mvn -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -B clean install --settings settings.xml -Dtest=IntegrationCaffe2Test
integrationGluonJobLinux:
stage: linux
image: registry.git.rwth-aachen.de/monticore/embeddedmontiarc/generators/emadl2cpp/integrationtests/mxnet:v0.0.3
image: registry.git.rwth-aachen.de/monticore/embeddedmontiarc/generators/emadl2cpp/integrationtests/mxnet:v0.0.4
script:
- mvn -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -B clean install --settings settings.xml -Dtest=IntegrationGluonTest
integrationTensorflowJobLinux:
stage: linux
image: registry.git.rwth-aachen.de/monticore/embeddedmontiarc/generators/emadl2cpp/integrationtests/tensorflow
script:
- mvn -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -B clean install --settings settings.xml -Dtest=IntegrationTensorflowTest
integrationPythonWrapperTest:
stage: linux
image: registry.git.rwth-aachen.de/monticore/embeddedmontiarc/generators/emadl2pythonwrapper/tests/mvn-swig:latest
script:
- mvn -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -B clean install --settings settings.xml -Dtest=IntegrationPythonWrapperTest
masterJobWindows:
stage: windows
script:
- mvn -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -B clean install --settings settings.xml -Dtest="GenerationTest,SymtabTest"
- mvn -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -B -U clean install --settings settings.xml -Dtest="GenerationTest,SymtabTest"
tags:
- Windows10
......
# (c) https://github.com/MontiCore/monticore
script:
- git checkout ${TRAVIS_BRANCH}
- mvn clean install cobertura:cobertura org.eluder.coveralls:coveralls-maven-plugin:report --settings "settings.xml"
......
{
"configurations": [
{
"type": "java",
"name": "CodeLens (Launch) - EMADLGeneratorCli",
"request": "launch",
"mainClass": "de.monticore.lang.monticar.emadl.generator.EMADLGeneratorCli",
"projectName": "embedded-montiarc-emadl-generator"
}
]
}
\ No newline at end of file
<!-- (c) https://github.com/MontiCore/monticore -->
![pipeline](https://git.rwth-aachen.de/monticore/EmbeddedMontiArc/generators/EMADL2CPP/badges/master/build.svg)
![coverage](https://git.rwth-aachen.de/monticore/EmbeddedMontiArc/generators/EMADL2CPP/badges/master/coverage.svg)
# EMADL2CPP
Generates CPP/Python code for EmbeddedMontiArcDL.
See example project [EMADL-Demo](https://git.rwth-aachen.de/thomas.timmermanns/EMADL-Demo) for more information on how the generated code can be used.
......@@ -19,7 +21,25 @@ See example project [EMADL-Demo](https://git.rwth-aachen.de/thomas.timmermanns/E
* Caffe2
* training - generated is Python code. Follow [ official instructions on Caffe2 site ](https://caffe2.ai/docs/getting-started.html?platform=ubuntu&configuration=prebuilt)
* Gluon
* See the scripts under Installation for better instructions, as an old caffe vversion is used that needs special considerations.
* Gluon
* Tensorflow
* training - generated is Python code.
* prediction - generated code is C++.
## Installation
The two bash scripts found under [installation scripts](https://git.rwth-aachen.de/monticore/EmbeddedMontiArc/generators/EMADL2CPP/tree/tensorflow_group/src/main/resources/installation_scripts)
should build and install all prerequisits for all backends as of 26.09.2019.
Note that the installation may take some time (hours) and you will need some disk space (> 60GB) for all backends. Also enough RAM or a big
enough swapspace is advisable (>10GB) for the installation of the cpp part of tensorflow. This scripts were tested with a completly clean Ubuntu 16.04,
without system updates installed. Using another Ubuntu version or installing other stuff, system updates included might/ have caused problems.
If you want to install the backends with CUDA GPU support(only MXNet/Gluon and Tensorflow, the used caffe2 version does not work with GPU support anymore),
you have to install CUDA 10.0(!!), CUDNN and NCCL (Obtainable from the nvidai webpage. You can follow their instructions.) inbetween the two scripts.
Furthermore you will have to change the pip commands for mxnet and tensorflow to the respective commented out parts.
Also docker images for the cpu version of each backend are provided at [Docker images](https://git.rwth-aachen.de/monticore/EmbeddedMontiArc/generators/EMADL2CPP/tree/tensorflow_group/src/test/resources/docker),
though some of these might be outdated.
### HowTo
1. Define a EMADL component containing architecture of a neural network and save it in a `.emadl` file. For more information on architecture language please refer to [CNNArchLang project](https://git.rwth-aachen.de/monticore/EmbeddedMontiArc/languages/CNNArchLang). An example of NN architecture:
......
<!-- (c) https://github.com/MontiCore/monticore -->
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
......@@ -8,18 +9,21 @@
<groupId>de.monticore.lang.monticar</groupId>
<artifactId>embedded-montiarc-emadl-generator</artifactId>
<version>0.3.0</version>
<version>0.3.8-SNAPSHOT</version>
<!-- == PROJECT DEPENDENCIES ============================================= -->
<properties>
<!-- .. SE-Libraries .................................................. -->
<emadl.version>0.2.6</emadl.version>
<CNNTrain.version>0.2.6</CNNTrain.version>
<cnnarch-mxnet-generator.version>0.2.14-SNAPSHOT</cnnarch-mxnet-generator.version>
<cnnarch-caffe2-generator.version>0.2.11-SNAPSHOT</cnnarch-caffe2-generator.version>
<cnnarch-gluon-generator.version>0.1.6</cnnarch-gluon-generator.version>
<emadl.version>0.2.11-SNAPSHOT</emadl.version>
<CNNTrain.version>0.3.9-SNAPSHOT</CNNTrain.version>
<cnnarch-generator.version>0.0.5-SNAPSHOT</cnnarch-generator.version>
<cnnarch-mxnet-generator.version>0.2.17-SNAPSHOT</cnnarch-mxnet-generator.version>
<cnnarch-caffe2-generator.version>0.2.14-SNAPSHOT</cnnarch-caffe2-generator.version>
<cnnarch-gluon-generator.version>0.2.10-SNAPSHOT</cnnarch-gluon-generator.version>
<cnnarch-tensorflow-generator.version>0.1.0-SNAPSHOT</cnnarch-tensorflow-generator.version>
<Common-MontiCar.version>0.0.14-20180704.113055-2</Common-MontiCar.version>
<embedded-montiarc-math-opt-generator>0.1.4</embedded-montiarc-math-opt-generator>
<!-- .. Libraries .................................................. -->
......@@ -61,25 +65,42 @@
</dependency>
<!-- MontiCore Dependencies -->
<dependency>
<groupId>de.monticore.lang.monticar</groupId>
<artifactId>embedded-montiarc-math-opt-generator</artifactId>
<version>${embedded-montiarc-math-opt-generator}</version>
</dependency>
<dependency>
<groupId>de.monticore.lang.monticar</groupId>
<artifactId>cnnarch-generator</artifactId>
<version>${cnnarch-generator.version}</version>
</dependency>
<dependency>
<groupId>de.monticore.lang.monticar</groupId>
<artifactId>cnnarch-mxnet-generator</artifactId>
<version>${cnnarch-mxnet-generator.version}</version>
</dependency>
<dependency>
<dependency>
<groupId>de.monticore.lang.monticar</groupId>
<artifactId>cnnarch-gluon-generator</artifactId>
<version>${cnnarch-gluon-generator.version}</version>
</dependency>
<dependency>
<groupId>de.monticore.lang.monticar</groupId>
<artifactId>common-monticar</artifactId>
<version>${Common-MontiCar.version}</version>
</dependency>
<dependency>
<groupId>de.monticore.lang.monticar</groupId>
<artifactId>cnnarch-tensorflow-generator</artifactId>
<version>${cnnarch-tensorflow-generator.version}</version>
</dependency>
<dependency>
<groupId>de.monticore.lang.monticar</groupId>
<artifactId>cnnarch-caffe2-generator</artifactId>
......@@ -136,7 +157,7 @@
</dependency>
</dependencies>
<!-- == PROJECT BUILD SETTINGS =========================================== -->
......@@ -249,6 +270,7 @@
<maxmem>256m</maxmem>
<!-- aggregated reports for multi-module projects -->
<aggregate>true</aggregate>
<check/>
</configuration>
</plugin>
</plugins>
......
<?xml version="1.0" encoding="UTF-8"?>
<!--
<!-- (c) https://github.com/MontiCore/monticore -->
******************************************************************************
MontiCAR Modeling Family, www.se-rwth.de
Copyright (c) 2017, Software Engineering Group at RWTH Aachen,
All rights reserved.
This project is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 3.0 of the License, or (at your option) any later version.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this project. If not, see <http://www.gnu.org/licenses/>.
*******************************************************************************
-->
<settings xmlns="http://maven.apache.org/SETTINGS/1.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0 http://maven.apache.org/xsd/settings-1.0.0.xsd">
......
(c) https://github.com/MontiCore/monticore
******************************************************************************
MontiCAR Modeling Family, www.se-rwth.de
Copyright (c) 2017, Software Engineering Group at RWTH Aachen,
All rights reserved.
The license generally applicable for this project
can be found under https://github.com/MontiCore/monticore.
This project is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 3.0 of the License, or (at your option) any later version.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this project. If not, see <http://www.gnu.org/licenses/>.
*******************************************************************************
/* (c) https://github.com/MontiCore/monticore */
package de.monticore.lang.monticar.emadl.generator;
import de.monticore.lang.monticar.cnnarch.CNNArchGenerator;
import de.monticore.lang.monticar.cnnarch.generator.CNNArchGenerator;
import de.monticore.lang.monticar.cnnarch.generator.CNNTrainGenerator;
import de.monticore.lang.monticar.cnnarch.gluongenerator.CNNArch2Gluon;
import de.monticore.lang.monticar.cnnarch.gluongenerator.CNNTrain2Gluon;
import de.monticore.lang.monticar.cnnarch.mxnetgenerator.CNNArch2MxNet;
import de.monticore.lang.monticar.cnnarch.caffe2generator.CNNArch2Caffe2;
import de.monticore.lang.monticar.cnnarch.mxnetgenerator.CNNTrain2MxNet;
import de.monticore.lang.monticar.cnnarch.caffe2generator.CNNArch2Caffe2;
import de.monticore.lang.monticar.cnnarch.caffe2generator.CNNTrain2Caffe2;
import de.monticore.lang.monticar.cnntrain.CNNTrainGenerator;
import de.monticore.lang.monticar.emadl.generator.reinforcementlearning.RewardFunctionCppGenerator;
import de.monticore.lang.monticar.cnnarch.tensorflowgenerator.CNNArch2Tensorflow;
import de.monticore.lang.monticar.cnnarch.tensorflowgenerator.CNNTrain2Tensorflow;
import java.util.Optional;
......@@ -40,10 +43,19 @@ public enum Backend {
}
@Override
public CNNTrainGenerator getCNNTrainGenerator() {
return new CNNTrain2Gluon();
return new CNNTrain2Gluon(new RewardFunctionCppGenerator());
}
},
TENSORFLOW{
@Override
public CNNArchGenerator getCNNArchGenerator() {
return new CNNArch2Tensorflow();
}
@Override
public CNNTrainGenerator getCNNTrainGenerator() {
return new CNNTrain2Tensorflow();
}
};
public abstract CNNArchGenerator getCNNArchGenerator();
public abstract CNNTrainGenerator getCNNTrainGenerator();
......@@ -54,9 +66,12 @@ public enum Backend {
case "CAFFE2":
return Optional.of(CAFFE2);
case "GLUON":
return Optional.of(GLUON);
case "TENSORFLOW":
return Optional.of(TENSORFLOW);
default:
return Optional.empty();
......@@ -67,10 +82,15 @@ public enum Backend {
switch (backend){
case CAFFE2:
return "CAFFE2";
case GLUON:
return "GLUON";
case TENSORFLOW:
return "TENSORFLOW";
default:
return "MXNET";
}
}
}
\ No newline at end of file
}
/**
*
* ******************************************************************************
* MontiCAR Modeling Family, www.se-rwth.de
* Copyright (c) 2017, Software Engineering Group at RWTH Aachen,
* All rights reserved.
*
* This project is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3.0 of the License, or (at your option) any later version.
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this project. If not, see <http://www.gnu.org/licenses/>.
* *******************************************************************************
*/
/* (c) https://github.com/MontiCore/monticore */
package de.monticore.lang.monticar.emadl.generator;
import de.monticore.ModelingLanguageFamily;
......@@ -25,6 +6,7 @@ import de.monticore.io.paths.ModelPath;
import de.monticore.lang.embeddedmontiarc.LogConfig;
import de.monticore.lang.embeddedmontiarc.helper.ConstantPortHelper;
import de.monticore.lang.monticar.emadl._symboltable.EMADLLanguage;
import de.monticore.lang.monticar.emadl.tagging.dltag.DataPathTagSchema;
import de.monticore.lang.monticar.enumlang._symboltable.EnumLangLanguage;
import de.monticore.lang.monticar.generator.cpp.converter.MathConverter;
import de.monticore.lang.monticar.generator.optimization.ThreadingOptimizer;
......@@ -58,6 +40,7 @@ public class EMADLAbstractSymtab {
TagInitTagSchema.registerTagTypes(tagging);
TagThresholdTagSchema.registerTagTypes(tagging);
TagDelayTagSchema.registerTagTypes(tagging);
DataPathTagSchema.registerTagTypes(tagging);
return tagging;
}
......
/**
*
* ******************************************************************************
* MontiCAR Modeling Family, www.se-rwth.de
* Copyright (c) 2017, Software Engineering Group at RWTH Aachen,
* All rights reserved.
*
* This project is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3.0 of the License, or (at your option) any later version.
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this project. If not, see <http://www.gnu.org/licenses/>.
* *******************************************************************************
*/
/* (c) https://github.com/MontiCore/monticore */
package de.monticore.lang.monticar.emadl.generator;
import de.se_rwth.commons.logging.Log;
......@@ -50,9 +31,10 @@ public class EMADLGeneratorCli {
.hasArg(true)
.required(false)
.build();
public static final Option OPTION_BACKEND = Option.builder("b")
.longOpt("backend")
.desc("deep-learning-framework backend. Options: MXNET, CAFFE2")
.desc("deep-learning-framework backend. Options: MXNET, CAFFE2, GLUON, TENSORFLOW")
.hasArg(true)
.required(false)
.build();
......
/* (c) https://github.com/MontiCore/monticore */
package de.monticore.lang.monticar.emadl.generator.reinforcementlearning;
import de.monticore.lang.embeddedmontiarc.embeddedmontiarc._symboltable.instanceStructure.EMAComponentInstanceSymbol;
import de.monticore.lang.monticar.cnnarch.gluongenerator.reinforcement.RewardFunctionSourceGenerator;
import de.monticore.lang.monticar.emadl.generator.EMADLAbstractSymtab;
import de.monticore.lang.monticar.generator.cpp.GeneratorEMAMOpt2CPP;
import de.monticore.lang.tagging._symboltable.TaggingResolver;
import de.se_rwth.commons.logging.Log;
import java.io.IOException;
import java.util.Optional;
public class RewardFunctionCppGenerator implements RewardFunctionSourceGenerator{
public RewardFunctionCppGenerator() {
}
@Override
public EMAComponentInstanceSymbol resolveSymbol(TaggingResolver taggingResolver, String rootModel) {
Optional<EMAComponentInstanceSymbol> instanceSymbol = taggingResolver
.<EMAComponentInstanceSymbol>resolve(rootModel, EMAComponentInstanceSymbol.KIND);
if (!instanceSymbol.isPresent()) {
Log.error("Generation of reward function is not possible: Cannot resolve component instance "
+ rootModel);
}
return instanceSymbol.get();
}
@Override
public void generate(EMAComponentInstanceSymbol componentInstanceSymbol, TaggingResolver taggingResolver,
String targetPath) {
GeneratorEMAMOpt2CPP generator = new GeneratorEMAMOpt2CPP();
generator.useArmadilloBackend();
generator.setGenerationTargetPath(targetPath);
try {
generator.generate(componentInstanceSymbol, taggingResolver);
} catch (IOException e) {
Log.error("Generation of reward function is not possible: " + e.getMessage());
}
}
@Override
public void generate(String modelPath, String rootModel, String targetPath) {
TaggingResolver taggingResolver = createTaggingResolver(modelPath);
EMAComponentInstanceSymbol instanceSymbol = resolveSymbol(taggingResolver, rootModel);
generate(instanceSymbol, taggingResolver, targetPath);
}
@Override
public TaggingResolver createTaggingResolver(final String modelPath) {
return EMADLAbstractSymtab.createSymTabAndTaggingResolver(modelPath);
}
}
/* (c) https://github.com/MontiCore/monticore */
#ifndef CNNTRANSLATOR_H
#define CNNTRANSLATOR_H
#include <armadillo>
#include <cassert>
#include <vector>
#include <utility>
#include <algorithm>
using namespace std;
using namespace arma;
......@@ -107,6 +111,46 @@ public:
return cubeMatrix;
}
static ivec translateToIntCol(const vector<float> &source, const vector<size_t> &shape){
assert(shape.size() == 1);
ivec column(shape[0]);
for(size_t i = 0; i < source.size(); i++){
column(i) = (size_t) source[i];
}
return column;
}
static imat translateToIntMat(const vector<float> &source, const vector<size_t> &shape){
assert(shape.size() == 2);
imat matrix(shape[1], shape[0]); //create transposed version of the matrix
int startPos = 0;
int endPos = matrix.n_rows;
const vector<size_t> columnShape = {matrix.n_rows};
for(size_t i = 0; i < matrix.n_cols; i++){
vector<float> colSource(&source[startPos], &source[endPos]);
matrix.col(i) = translateToIntCol(colSource, columnShape);
startPos = endPos;
endPos += matrix.n_rows;
}
return matrix.t();
}
static icube translateToIntCube(const vector<float> &source, const vector<size_t> &shape){
assert(shape.size() == 3);
icube cubeMatrix(shape[1], shape[2], shape[0]);
const int matrixSize = shape[1] * shape[2];
const vector<size_t> matrixShape = {shape[1], shape[2]};
int startPos = 0;
int endPos = matrixSize;
for(size_t i = 0; i < cubeMatrix.n_slices; i++){
vector<float> matrixSource(&source[startPos], &source[endPos]);
cubeMatrix.slice(i) = translateToIntMat(matrixSource, matrixShape);
startPos = endPos;
endPos += matrixSize;
}
return cubeMatrix;
}
template<typename T> static vector<size_t> getShape(const Col<T> &source){
return {source.n_elem};
}
......
#!/bin/bash
############################ Part 2/2 ######################################
# This script should help you to get ready to do machine learning #
# at SE at RWTH, installing all backends (mxnet/gluon, tensorflow, caffe2. #
# - Jonas Eckhardt & Julian Steinsberger-Dührßen - #
############################################################################
# It is based both on instrcutions and docker files from the git
# repos as well as our own work
############################################################################
# Requirements: #
############################################################################
# -Git set up to work with RWTH Gitlab
# -Virgin Ubuntu 16.04.(No touching, no other version) and not isntalling the system updates
# -Time: Installation may need more than one hour. It may prompt you
# for your passowrd multiple times
# -Disable Screensaver: It just saves you some time
# - > 10GB RAM or a swapfile so you have ram+swap > 10GB
# - > 60GB disk space
###########################################################################
# Usage: #
###########################################################################
# chmod u+x installer_bomb.sh
# ./installer_bomb.sh
# Dont use twice. It will fill your ~/.bashrc with garbage#!/bin/bash
sudo apt install -y --no-install-recommends cuda-command-line-tools-10-0
#Install MXNet
pip install mxnet --user
#for cuda support: use mxnet-cu100 --user instead
git clone https://github.com/apache/incubator-mxnet.git mxnet-source
cd mxnet-source
git checkout tags/1.4.0
cd ..
sudo cp -r mxnet-source/include/mxnet /usr/include/mxnet
#Test mxnet
git clone git@git.rwth-aachen.de:monticore/EmbeddedMontiArc/applications/mnistcalculator.git
cd mnistcalculator
cd mxnet
./build.sh
cd ..
cd ..
#Install OpenCV
git clone https://github.com/opencv/opencv.git
cd opencv
git checkout 3.3.1
cd ..
git clone https://github.com/opencv/opencv_contrib.git
cd opencv_contrib
git checkout 3.3.1
cd ..
cd opencv
mkdir build
cd build
cmake -D CMAKE_BUILD_TYPE=RELEASE -D CMAKE_INSTALL_PREFIX=/usr/local \
-D INSTALL_C_EXAMPLES=ON \
-D INSTALL_PYTHON_EXAMPLES=ON \
-D WITH_TBB=ON \
-D WITH_V4L=ON \
-D WITH_QT=ON \
-D WITH_OPENGL=ON \
-D OPENCV_EXTRA_MODULES_PATH=../../opencv_contrib/modules \
-D BUILD_EXAMPLES=ON ..
make -j$(nproc)
sudo make install
sudo sh -c 'echo "/usr/local/lib" >> /etc/ld.so.conf.d/opencv.conf'
sudo ldconfig
#Install Tensorflow
pip install --upgrade tensorflow --user
#for cuda support: pip install --upgrade tensorflow-gpu --user
git clone https://github.com/FloopCZ/tensorflow_cc.git
#Install bazel
sudo wget -P /tmp https://github.com/bazelbuild/bazel/releases/download/0.21.0/bazel-0.21.0-installer-linux-x86_64.sh
sudo chmod +x /tmp/bazel-0.21.0-installer-linux-x86_64.sh
sudo /tmp/bazel-0.21.0-installer-linux-x86_64.sh
sudo rm /tmp/bazel-0.21.0-installer-linux-x86_64.sh
sudo apt-get -y clean
sudo updatedb
cd tensorflow_cc/tensorflow_cc/
mkdir build
cd build
cmake -DTENSORFLOW_STATIC=OFF -DTENSORFLOW_SHARED=ON ..
make
sudo rm -rf /home/tensorflow_cc/.cache
make install
cd ..
sudo rm -rf build
\ No newline at end of file
#!/bin/bash
############################ Part 1/2 ######################################
# This script should help you to get ready to do machine learning #
# at SE at RWTH, installing all backends (mxnet/gluon, tensorflow, caffe2. #
# - Jonas Eckhardt & Julian Steinsberger-Dührßen - #
############################################################################
# It is based both on instrcutions and docker files from the git
# repos as well as our own work
############################################################################
# Requirements: #
############################################################################
# -Git set up to work with RWTH Gitlab
# -Virgin Ubuntu 16.04.(No touching, no other version) and not isntalling the system updates
# -Time: Installation may need more than one hour. It may prompt you
# for your passowrd multiple times
# -Disable Screensaver: It just saves you some time
# - > 10GB RAM or a swapfile so you have ram+swap > 10GB
# - > 60GB disk space
###########################################################################
# Usage: #
###########################################################################
# chmod u+x installer_bomb.sh
# ./installer_bomb.sh
# Dont use twice. It will fill your ~/.bashrc with garbage
sudo apt update
sudo apt upgrade -y
sudo apt install -y --no-install-recommends maven libopencv-dev\
python-pip openjdk-8-jdk git wget python python3-numpy gcc build-essential cmake \
liblapack-dev libblas-dev libopenblas-dev libboost-dev libarmadillo-dev build-essential \
libgoogle-glog-dev libgtest-dev libiomp-dev libleveldb-dev liblmdb-dev \
libopenmpi-dev libsnappy-dev libprotobuf-dev openmpi-bin openmpi-doc python-numpy swig
sudo apt install -y --no-install-recommends protobuf-compiler python-dev python-setuptools \
libgflags-dev checkinstall pkg-config yasm gfortran libjpeg8-dev libjasper-dev libpng12-dev \
libtiff5-dev libavcodec-dev libavformat-dev libswscale-dev libdc1394-22-dev libxine2-dev \
libgstreamer-plugins-base0.10-dev qt5-default libgtk2.0-dev libtbb-dev libatlas-base-dev \
libfaac-dev libmp3lame-dev libtheora-dev libvorbis-dev libxvidcore-dev \
libopencore-amrnb-dev libopencore-amrwb-dev x264 v4l-utils libgphoto2-dev libeigen3-dev \
libhdf5-dev doxygen curl unzip autoconf autogen libtool mlocate zlib1g-dev g++ sudo zip automake
#Fix JDK version
sudo update-alternatives --config java
pip install --upgrade pip --user
pip install numpy opencv-python six lmdb graphviz pyyaml future typing --user
pip install --no-deps h5py --user
#prepare path variables
echo "export PATH=\$PATH:\"/usr/local/lib\"" >> ~/.bashrc
echo "export PYTHON_PATH=\"/usr/local:$(pwd)/pytorch/build:/usr/bin/python:/usr/local/lib\"">> ~/.bashrc
echo "export LD_LIBRARY_PATH=\"/usr/local/lib\"">> ~/.bashrc
source ~/.bashrc
#Install caffee2
git clone --recursive https://github.com/pytorch/pytorch.git
cd pytorch
git checkout v0.4.0
git -c submodule.'third_party/nervanagpu'.update=none submodule update --init
sudo rm -r build
mkdir build
cd build
cmake -DUSE_MPI=OFF ..
sudo make clean install
cd ..
sudo FULL_CAFFE2=1 python setup.py clean install
sudo cp -r /usr/local/lib/libcaffe2.so /usr/lib/
cd ..
#Fix protocbuf
pip2 install protobuf==3.5.1 --user
sudo snap install --classic notepadqq
sudo snap install pycharm-community --classic
\ No newline at end of file
/**
*
* ******************************************************************************
* MontiCAR Modeling Family, www.se-rwth.de
* Copyright (c) 2017, Software Engineering Group at RWTH Aachen,
* All rights reserved.
*
* This project is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3.0 of the License, or (at your option) any later version.
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this project. If not, see <http://www.gnu.org/licenses/>.
* *******************************************************************************
*/
/* (c) https://github.com/MontiCore/monticore */
package de.monticore.lang.monticar.emadl;
import de.monticore.lang.monticar.emadl.generator.EMADLAbstractSymtab;
......
/**
*
* ******************************************************************************
* MontiCAR Modeling Family, www.se-rwth.de
* Copyright (c) 2017, Software Engineering Group at RWTH Aachen,
* All rights reserved.
*
* This project is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3.0 of the License, or (at your option) any later version.
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this project. If not, see <http://www.gnu.org/licenses/>.
* *******************************************************************************
*/
/* (c) https://github.com/MontiCore/monticore */
package de.monticore.lang.monticar.emadl;
import de.monticore.lang.monticar.emadl.generator.Backend;
import de.monticore.lang.monticar.emadl.generator.EMADLGenerator;
import de.monticore.lang.monticar.emadl.generator.EMADLGeneratorCli;
import de.se_rwth.commons.logging.Log;
import freemarker.template.TemplateException;
import org.junit.Before;
import org.junit.Test;
import java.io.IOException;
import java.nio.charset.Charset;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Arrays;
import java.util.List;
import static junit.framework.TestCase.assertTrue;
import static org.junit.Assert.assertFalse;
import org.junit.Ignore;
@Ignore
public class IntegrationCaffe2Test extends IntegrationTest {
public IntegrationCaffe2Test() {
super("CAFFE2", "39253EC049D4A4E5FA0536AD34874B9D#1DBAEE1B1BD83FB7CB5F70AE91B29638#13D139510DC5681639AA91D7250288D3#1A42D4842D0664937A9F6B727BD60CEF");
......
/**
*
* ******************************************************************************
* MontiCAR Modeling Family, www.se-rwth.de
* Copyright (c) 2017, Software Engineering Group at RWTH Aachen,
* All rights reserved.
*
* This project is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3.0 of the License, or (at your option) any later version.
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this project. If not, see <http://www.gnu.org/licenses/>.
* *******************************************************************************
*/
/* (c) https://github.com/MontiCore/monticore */
package de.monticore.lang.monticar.emadl;
import de.monticore.lang.monticar.emadl.generator.EMADLGeneratorCli;
import de.se_rwth.commons.logging.Log;
import org.junit.Ignore;
import org.junit.Test;
import java.nio.file.Files;
import java.nio.file.NoSuchFileException;
import java.nio.file.Path;
import java.nio.file.Paths;
import static junit.framework.TestCase.assertTrue;
import static org.junit.Assert.assertFalse;
public class IntegrationGluonTest extends IntegrationTest {
public IntegrationGluonTest() {
super("GLUON", "39253EC049D4A4E5FA0536AD34874B9D#1DBAEE1B1BD83FB7CB5F70AE91B29638#C4C23549E737A759721D6694C75D9771#5AF0CE68E408E8C1F000E49D72AC214A");
}
@Test
public void testMultipleStreams() {
Log.getFindings().clear();
deleteHashFile(Paths.get("./target/generated-sources-emadl/MultipleStreams.training_hash"));
String[] args = {"-m", "src/test/resources/models/", "-r", "MultipleStreams", "-b", "GLUON"};
EMADLGeneratorCli.main(args);
assertTrue(Log.getFindings().isEmpty());
}
@Ignore
@Test
public void testRNNencdec() {
Log.getFindings().clear();
deleteHashFile(Paths.get("./target/generated-sources-emadl/rnnencdec/Network.training_hash"));
String[] args = {"-m", "src/test/resources/models", "-r", "rnnencdec.Main", "-b", "GLUON"};
EMADLGeneratorCli.main(args);
assertTrue(Log.getFindings().isEmpty());
}
@Ignore
@Test
public void testRNNsearch() {
Log.getFindings().clear();
deleteHashFile(Paths.get("./target/generated-sources-emadl/rnnsearch/Network.training_hash"));
String[] args = {"-m", "src/test/resources/models", "-r", "rnnsearch.Main", "-b", "GLUON"};
EMADLGeneratorCli.main(args);
assertTrue(Log.getFindings().isEmpty());
}
@Test
public void testShowAttendTell() {
Log.getFindings().clear();
deleteHashFile(Paths.get("./target/generated-sources-emadl/showAttendTell/Show_attend_tell.training_hash"));
String[] args = {"-m", "src/test/resources/models", "-r", "showAttendTell.Main", "-b", "GLUON"};
EMADLGeneratorCli.main(args);
assertTrue(Log.getFindings().isEmpty());
}
private void deleteHashFile(Path hashFile) {
try {
Files.delete(hashFile);
}
catch (NoSuchFileException e) {
}
catch(Exception e) {
assertFalse("Could not delete hash file", true);
}
}
}
/**
*
* ******************************************************************************
* MontiCAR Modeling Family, www.se-rwth.de
* Copyright (c) 2017, Software Engineering Group at RWTH Aachen,
* All rights reserved.
*
* This project is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3.0 of the License, or (at your option) any later version.
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this project. If not, see <http://www.gnu.org/licenses/>.
* *******************************************************************************
*/
/* (c) https://github.com/MontiCore/monticore */
package de.monticore.lang.monticar.emadl;
import de.monticore.lang.monticar.emadl.generator.Backend;
import de.monticore.lang.monticar.emadl.generator.EMADLGenerator;
import de.monticore.lang.monticar.emadl.generator.EMADLGeneratorCli;
import de.se_rwth.commons.logging.Log;
import freemarker.template.TemplateException;
import org.junit.Before;
import org.junit.Test;
import java.io.IOException;
import java.nio.charset.Charset;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Arrays;
import java.util.List;
import static junit.framework.TestCase.assertTrue;
import static org.junit.Assert.assertFalse;
public class IntegrationMXNetTest extends IntegrationTest {
public IntegrationMXNetTest() {
super("MXNET", "39253EC049D4A4E5FA0536AD34874B9D#1DBAEE1B1BD83FB7CB5F70AE91B29638#C4C23549E737A759721D6694C75D9771#5AF0CE68E408E8C1F000E49D72AC214A");
......
/* (c) https://github.com/MontiCore/monticore */
package de.monticore.lang.monticar.emadl;
import de.monticore.lang.monticar.emadl.generator.EMADLGeneratorCli;
import de.se_rwth.commons.logging.Finding;
import de.se_rwth.commons.logging.Log;
import org.junit.Test;
import java.nio.file.Paths;
import java.util.Arrays;
import java.util.stream.Collectors;
import static junit.framework.TestCase.assertTrue;
import static org.junit.Assume.assumeFalse;
public class IntegrationPythonWrapperTest extends AbstractSymtabTest {
@Test
public void testGluonReinforcementModelRosEnvironment() {
assumeFalse(System.getProperty("os.name").toLowerCase().startsWith("win"));
Log.getFindings().clear();
String[] args = {"-m", "src/test/resources/models/reinforcementModel", "-r", "torcs.agent.TorcsAgent", "-b", "GLUON", "-f", "n", "-c", "n"};
EMADLGeneratorCli.main(args);
assertTrue(Log.getFindings().stream().filter(Finding::isError).collect(Collectors.toList()).isEmpty());
checkFilesAreEqual(
Paths.get("./target/generated-sources-emadl"),
Paths.get("./src/test/resources/target_code/gluon/reinforcementModel/torcs"),
Arrays.asList(
"CMakeLists.txt",
"CNNBufferFile.h",
"torcs_agent_torcsAgent.cpp",
"torcs_agent_torcsAgent.h",
"torcs_agent_torcsAgent_dqn.h",
"torcs_agent_torcsAgent_policy.h",
"CNNCreator_torcs_agent_torcsAgent_dqn.py",
"CNNNet_torcs_agent_torcsAgent_dqn.py",
"CNNPredictor_torcs_agent_torcsAgent_dqn.h",
"CNNTrainer_torcs_agent_torcsAgent_dqn.py",
"CNNTranslator.h",
"HelperA.h",
"start_training.sh",
"reward/CMakeLists.txt",
"reward/HelperA.h",
"reward/torcs_agent_dqn_reward.cpp",
"reward/torcs_agent_dqn_reward.h",
"reward/pylib/CMakeLists.txt",
"reward/pylib/torcs_agent_dqn_reward_executor.cpp",
"reward/pylib/torcs_agent_dqn_reward_executor.h",
"reward/pylib/torcs_agent_dqn_reward_executor.i",
"reward/pylib/armanpy/armanpy.hpp",
"reward/pylib/armanpy/armanpy.i",
"reward/pylib/armanpy/armanpy_1d.i",
"reward/pylib/armanpy/armanpy_2d.i",
"reward/pylib/armanpy/armanpy_3d.i",
"reward/pylib/armanpy/numpy.i",
"reinforcement_learning/__init__.py",
"reinforcement_learning/strategy.py",
"reinforcement_learning/agent.py",
"reinforcement_learning/environment.py",
"reinforcement_learning/replay_memory.py",
"reinforcement_learning/util.py",
"reinforcement_learning/cnnarch_logger.py"
)
);
/*
assertTrue(Paths.get(
"./target/generated-sources-emadl/reinforcement_learning/_torcs_agent_dqn_reward_executor.so")
.toFile().exists());
assertTrue(Paths.get(
"./target/generated-sources-emadl/reinforcement_learning/torcs_agent_dqn_reward_executor.py")
.toFile().exists());
*/
}
@Test
public void testTorcsTD3() {
Log.getFindings().clear();
String[] args = {"-m", "src/test/resources/models/reinforcementModel/torcs_td3", "-r", "torcs.agent.TorcsAgent", "-b", "GLUON", "-f", "n", "-c", "n"};
EMADLGeneratorCli.main(args);
assertTrue(Log.getFindings().stream().filter(Finding::isError).collect(Collectors.toList()).isEmpty());
checkFilesAreEqual(
Paths.get("./target/generated-sources-emadl"),
Paths.get("./src/test/resources/target_code/gluon/reinforcementModel/torcs_td3"),
Arrays.asList(
"CMakeLists.txt",
"CNNBufferFile.h",
"torcs_agent_torcsAgent.cpp",
"torcs_agent_torcsAgent.h",
"torcs_agent_torcsAgent_actor.h",
"CNNCreator_torcs_agent_torcsAgent_actor.py",
"CNNNet_torcs_agent_torcsAgent_actor.py",
"CNNPredictor_torcs_agent_torcsAgent_actor.h",
"CNNTrainer_torcs_agent_torcsAgent_actor.py",
"start_training.sh",
"reward/CMakeLists.txt",
"reward/torcs_agent_network_reward.cpp",
"reward/torcs_agent_network_reward.h",
"reward/pylib/CMakeLists.txt",
"reward/pylib/torcs_agent_network_reward_executor.cpp",
"reward/pylib/torcs_agent_network_reward_executor.h",
"reward/pylib/torcs_agent_network_reward_executor.i",
"reward/pylib/armanpy/armanpy.hpp",
"reward/pylib/armanpy/armanpy.i",
"reward/pylib/armanpy/armanpy_1d.i",
"reward/pylib/armanpy/armanpy_2d.i",
"reward/pylib/armanpy/armanpy_3d.i",
"reward/pylib/armanpy/numpy.i",
"reinforcement_learning/__init__.py",
"reinforcement_learning/strategy.py",
"reinforcement_learning/agent.py",
"reinforcement_learning/environment.py",
"reinforcement_learning/replay_memory.py",
"reinforcement_learning/util.py",
"reinforcement_learning/cnnarch_logger.py",
"reinforcement_learning/CNNCreator_torcs_agent_network_torcsCritic.py",
"reinforcement_learning/CNNNet_torcs_agent_network_torcsCritic.py"
)
);
}
}
/**
*
* ******************************************************************************
* MontiCAR Modeling Family, www.se-rwth.de
* Copyright (c) 2017, Software Engineering Group at RWTH Aachen,
* All rights reserved.
*
* This project is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3.0 of the License, or (at your option) any later version.
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this project. If not, see <http://www.gnu.org/licenses/>.
* *******************************************************************************
*/
package de.monticore.lang.monticar.emadl;
import de.monticore.lang.monticar.emadl.generator.EMADLGeneratorCli;
import de.se_rwth.commons.logging.Log;
import org.junit.Ignore;
import org.junit.Test;
import java.nio.file.Files;
import java.nio.file.NoSuchFileException;
import java.nio.file.Path;
import java.nio.file.Paths;
import static junit.framework.TestCase.assertTrue;
import static org.junit.Assert.assertFalse;
@Ignore
public class IntegrationTensorflowTest extends IntegrationTest {
private Path multipleStreamsHashFile = Paths.get("./target/generated-sources-emadl/MultipleStreams.training_hash");
public IntegrationTensorflowTest() {