Commit d247f9e8 authored by Evgeny Kusmenko's avatar Evgeny Kusmenko

Merge branch 'lab_calculator' into 'master'

Lab calculator

See merge request !2
parents 9a28adc8 71a7240a
Pipeline #122841 passed with stage
in 19 minutes and 19 seconds
#
#
# ******************************************************************************
# MontiCAR Modeling Family, www.se-rwth.de
# Copyright (c) 2017, Software Engineering Group at RWTH Aachen,
# All rights reserved.
#
# This project is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 3.0 of the License, or (at your option) any later version.
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this project. If not, see <http://www.gnu.org/licenses/>.
# *******************************************************************************
#
stages:
- linux
TestMXNET:
stage: linux
image: registry.git.rwth-aachen.de/monticore/embeddedmontiarc/applications/mnistcalculator/mxnet:v0.0.3
script:
- cd mxnet
- ./prepare.sh
- ./build.sh
- RES=$(./build/src/cpp/DigitCalculator resources/images/1.png resources/images/2.png resources/images/3.png resources/images/4.png resources/images/5.png resources/images/6.png)
- "if [[ $RES != *\"SUM: 579\"* ]]; then echo \"Wrong result:\" $RES; exit 1; fi;"
TestCAFFE2:
stage: linux
image: registry.git.rwth-aachen.de/monticore/embeddedmontiarc/applications/mnistcalculator/caffe2:v0.0.2
script:
- cd caffe2
- ./build.sh
- RES=$(./build/src/cpp/DigitCalculator resources/images/2.jpg resources/images/3.jpg resources/images/4.png resources/images/6.png resources/images/2.jpg resources/images/3.jpg)
- "if [[ $RES != *\"SUM: 857\"* ]]; then echo \"Wrong result:\" $RES; exit 1; fi;"
# MNISTCalculator
## Prerequisites
1. Ubuntu Linux 16.04 LTS or 18.04 LTS (experimental)
2. Deep Learning Framework **Caffe2**. [Follow the instructions from this link](https://git.rwth-aachen.de/monticore/EmbeddedMontiArc/generators/CNNArch2Caffe2#ubuntu).
3. Deep Learning Framework **MXNet**.
4. Armadillo (at least armadillo version 6.600 must be used) [Official instructions at Armadillo Website](http://arma.sourceforge.net/download.html).
4. OpenCV
## MXNet and Caffe2
For detailed instructions on how to run the MXNet and Caffe2 example, see the README in the corresponding folder.
\ No newline at end of file
train.log
target/
resources/training_data
model/
build/
\ No newline at end of file
# MNISTCalculator Caffe2
## Prerequisites
1. Ubuntu Linux 16.04 LTS or 18.04 LTS (experimental)
2. Deep Learning Framework **Caffe2**. [Follow the instructions from this link](https://git.rwth-aachen.de/monticore/EmbeddedMontiArc/generators/CNNArch2Caffe2#ubuntu).
3. Armadillo (at least armadillo version 6.600 must be used) [Official instructions at Armadillo Website](http://arma.sourceforge.net/download.html).
4. OpenCV
## How to Run
Generate and build the code for the EMADL model by executing:
```
bash build.sh
```
Finally, run the calculator as follows:
```
./build/src/cpp/DigitCalculator test_img/2.jpg test_img/3.jpg test_img/4.png test_img/6.png test_img/2.jpg test_img/3.jpg
```
You can try different images from the provided `/test_img` directory or even create your own images for testing. Note that the application program DigitCalculator will automatically rescale the given input image to the size 28x28.
## Troubleshooting Help
ERROR: HelperA.h:79:28: error: ‘sqrtmat’ was not declared in this scope.
FIX:
Copy compiled armadillo lib and include files to usr/lib and usr/include respectively. Replace YOUR_ARMADILLO_REPOSITORY and VERSION (e.g. 8.500.1) with your corresponding information:
```
cd YOUR_ARMADILLO_REPOSITORY/armadillo-VERSION
sudo cp libarmadillo* /usr/lib
cd YOUR_ARMADILLO_REPOSITORY/armadillo-VERSION/include
sudo cp -r * /usr/include
rm -r target/;
java -jar embedded-montiarc-emadl-generator-0.2.9-jar-with-dependencies.jar -m src/emadl/models/ -r cNNCalculator.Connector -o target -b CAFFE2
rm -rf target
java -jar ../embedded-montiarc-emadl-generator-0.2.13-jar-with-dependencies.jar -m src/emadl/models/ -r cNNCalculator.Connector -o target -b CAFFE2
rm -rf build
mkdir build && cd build
......
FROM registry.git.rwth-aachen.de/monticore/embeddedmontiarc/generators/emadl2cpp/integrationtests/caffe2:v0.0.5
RUN apt-get update
RUN apt-get install -y libopenblas-dev
\ No newline at end of file
configuration Network{
num_epoch:11
num_epoch:10
batch_size:64
context:gpu
context:cpu
eval_metric:accuracy
optimizer:adam{
learning_rate:0.001
......
rm -r target/;
java -jar embedded-montiarc-emadl-generator-0.2.5-SNAPSHOT-jar-with-dependencies.jar -m src/emadl/models/ -r cNNCalculator.Connector -o target -b MXNET
rm -rf build
mkdir build && cd build
echo "Building DigitClassifier.."
cmake ..
#cp -r /home/christopher/Documents/Master/Semester2/Praktikum/mxnet/include/mxnet/ ../target/
make
\ No newline at end of file
echo "-- Generating files and training Models:"
java -jar embedded-montiarc-emadl-generator-0.2.5-SNAPSHOT-jar-with-dependencies.jar -m src/emadl/models -r cNNCalculator.Connector -o target -b MXNET
mkdir 0
mv *-num0.png 0/.
mkdir 1
mv *-num1.png 1/.
mkdir 2
mv *-num2.png 2/.
mkdir 3
mv *-num3.png 3/.
mkdir 4
mv *-num4.png 4/.
mkdir 5
mv *-num5.png 5/.
mkdir 6
mv *-num6.png 6/.
mkdir 7
mv *-num7.png 7/.
mkdir 8
mv *-num8.png 8/.
mkdir 9
mv *-num9.png 9/.
from __future__ import print_function
from __future__ import division
import h5py
import numpy as np
import cv2
import os
import argparse
import errno
import random
import sys
def create_img_list(name, data_path):
dir_name = data_path + "/" + name
image_paths = []
image_class_indices = []
print(dir_name)
for class_index_name in os.listdir(dir_name):
class_dir_path = dir_name + "/" + class_index_name
if os.path.isdir(class_dir_path):
for image_name in os.listdir(class_dir_path):
image_path = class_dir_path + "/" + image_name
image_paths.append(image_path)
class_index = float(class_index_name)
image_class_indices.append(class_index)
return image_paths, image_class_indices
def create_h5_from_list(image_paths, image_class_indices, target_dir, target_file_name, input_port_name, output_port_name, shuffle=True):
img = cv2.imread(image_paths[0])
t_img = np.transpose(img, (2,0,1)).astype(np.float32)
#t_img = t_img[-1:,:,:]
#print(t_img)
channels = t_img.shape[0]
height = t_img.shape[1]
width = t_img.shape[2]
data_size = len(image_paths)
target_file = target_dir + "/" + target_file_name + ".h5"
if os.path.isfile(target_file):
print("File", target_file, "already exists. Skipping data file creation.")
return
try:
os.makedirs(target_dir)
except OSError as e:
if e.errno != errno.EEXIST:
raise
if shuffle:
combined = list(zip(image_paths, image_class_indices))
random.shuffle(combined)
image_paths[:], image_class_indices[:] = zip(*combined)
print("Creating " + target_file + " (images:" + str(data_size) + ", channels:" + str(channels) + ", height:" + str(height) + ", width:" + str(width) + "):")
with h5py.File(target_file, "w") as ofile:
in_dset = ofile.create_dataset(input_port_name, (data_size,channels, height, width), dtype=np.float32)
out_dset = ofile.create_dataset(output_port_name + "_label", (data_size,), dtype=np.float32)
for i in range(data_size):
img = cv2.imread(image_paths[i])
t_img = np.transpose(img, (2,0,1)).astype(np.float32)
#t_img = t_img[-1:,:,:]
in_dset[i] = t_img
out_dset[i] = image_class_indices[i]
#print progress
if i % 100 == 0:
percentage = 100*i / data_size
sys.stdout.write("\r{:0.1f}%".format(percentage))
sys.stdout.flush()
sys.stdout.write("\r100.0%\n")
sys.stdout.flush()
if __name__ == "__main__":
parser = argparse.ArgumentParser(description='Translate image directories into hdf5 training sets for EMADL.')
parser.add_argument("--in_port", action="store", dest="in_port", default="data")
parser.add_argument("--out_port", action="store", dest="out_port", default="softmax")
parser.add_argument("--data_path", action="store", dest="data_path", default=".")
parser.add_argument("--target_path", action="store", dest="target_path", default=".")
args = parser.parse_args()
for file_name in os.listdir(args.data_path):
if file_name == "train":
image_paths, image_class_indices = create_img_list(file_name, args.data_path)
create_h5_from_list(image_paths, image_class_indices, args.target_path, file_name, args.in_port, args.out_port)
if file_name == "test":
image_paths, image_class_indices = create_img_list(file_name, args.data_path)
create_h5_from_list(image_paths, image_class_indices, args.target_path, file_name, args.in_port, args.out_port)
mkdir 0
mv *-num0.png 0/.
mkdir 1
mv *-num1.png 1/.
mkdir 2
mv *-num2.png 2/.
mkdir 3
mv *-num3.png 3/.
mkdir 4
mv *-num4.png 4/.
mkdir 5
mv *-num5.png 5/.
mkdir 6
mv *-num6.png 6/.
mkdir 7
mv *-num7.png 7/.
mkdir 8
mv *-num8.png 8/.
mkdir 9
mv *-num9.png 9/.
# Calculator MXNET
## Prerequisites
1. Ubuntu Linux 16.04 LTS or 18.04 LTS (experimental)
2. Deep Learning Framework **MXNet**
3. Armadillo (at least armadillo version 6.600 must be used) [Official instructions at Armadillo Website](http://arma.sourceforge.net/download.html).
4. OpenCV
## Prepare the data
To create the data for training, execute:
```
bash ./prepare.sh
```
## How to Run
Generate and build the code for the EMADL model by executing:
```
bash build.sh
```
Finally, run the calculator as follows:
```
./build/src/cpp/DigitCalculator resources/images/1.png resources/images/2.png resources/images/3.png resources/images/4.png resources/images/5.png resources/images/6.png
```
You can try different images from the provided `/resources/images` directory or even create your own images for testing. Note that the application program DigitCalculator will automatically rescale the given input image to the size 32x32.
Note: Currently, the mxnet example uses CIFAR-10 data, not MNIST!
## Troubleshooting Help
ERROR: HelperA.h:79:28: error: ‘sqrtmat’ was not declared in this scope.
FIX:
Copy compiled armadillo lib and include files to usr/lib and usr/include respectively. Replace YOUR_ARMADILLO_REPOSITORY and VERSION (e.g. 8.500.1) with your corresponding information:
```
cd YOUR_ARMADILLO_REPOSITORY/armadillo-VERSION
sudo cp libarmadillo* /usr/lib
cd YOUR_ARMADILLO_REPOSITORY/armadillo-VERSION/include
sudo cp -r * /usr/include
MXNET_PATH=$(python -c "import mxnet; print(mxnet.__file__)")
MXNET_FOLDER=$(dirname $MXNET_PATH)
echo $MXNET_FOLDER
if [ ! -f $MXNET_FOLDER/libmxnet.so ]; then
echo "libmxnet.so not found at default location" $MXNET_FOLDER
echo "It should be there if the python mxnet package is installed"
echo "Either fix the installation, or adapt the ./build.sh script to locate libmxnet.so correctly"
exit 1
fi
rm -rf target
java -jar ../embedded-montiarc-emadl-generator-0.2.13-jar-with-dependencies.jar -m src/emadl/models/ -r cNNCalculator.Connector -o target -b MXNET
rm -rf build
mkdir build && cd build
echo "Building DigitClassifier.."
cmake -D MXNET_PATH=$MXNET_FOLDER/libmxnet.so ..
make
\ No newline at end of file
FROM registry.git.rwth-aachen.de/monticore/embeddedmontiarc/generators/emadl2cpp/integrationtests/mxnet:v0.0.3
RUN pip install opencv-python
# Install Open CV - Warning, this takes absolutely forever
RUN apt-get update && apt-get install -y libgdal-dev
RUN mkdir -p ~/opencv cd ~/opencv && \
wget https://github.com/Itseez/opencv/archive/3.0.0.zip && \
unzip 3.0.0.zip && \
rm 3.0.0.zip && \
mv opencv-3.0.0 OpenCV
RUN apt-get install -y qtbase5-dev
RUN apt-get install -y build-essential
ENV PATH="${PATH}:/usr/include/"
RUN cd OpenCV && \
mkdir build && \
cd build && \
cmake \
-DWITH_QT=ON \
-DWITH_OPENGL=ON \
-DFORCE_VTK=ON \
-DWITH_TBB=ON \
-DWITH_GDAL=ON \
-DWITH_XINE=ON \
-DBUILD_EXAMPLES=ON \
-D ENABLE_PRECOMPILED_HEADERS=OFF ..
RUN cd OpenCV/build && make -j4
RUN cd OpenCV/build && make install
RUN cd OpenCV/build && ldconfig
RUN apt-get install -y libopenblas-dev
\ No newline at end of file
......@@ -2,13 +2,14 @@ cmake_minimum_required(VERSION 3.5)
project(DigitCalculator CXX)
set(CMAKE_CXX_STANDARD 11)
option(MXNET_PATH "Path to libmxnet.so")
add_executable(DigitCalculator DigitCalculator.cpp)
set_target_properties(DigitCalculator PROPERTIES LINKER_LANGUAGE CXX)
find_package(OpenCV REQUIRED)
target_link_libraries(DigitCalculator "/home/christopher/Documents/Master/Semester2/Praktikum/mxnet/lib/libmxnet.so")
target_link_libraries(DigitCalculator ${MXNET_PATH})
set(INCLUDE_DIRS ${INCLUDE_DIRS} ${OpenCV_INCLUDE_DIRS})
set(LIBS ${LIBS} ${OpenCV_LIBS})
target_link_libraries(DigitCalculator cNNCalculator_connector ${LIBS} -lblas -lopenblas)
......
configuration Network{
num_epoch:5
num_epoch:10
batch_size:100
normalize:true
context:gpu
context:cpu
load_checkpoint:false
optimizer:sgd{
learning_rate:0.2
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment