Commit 9e4b6408 authored by Yuyuan Liu's avatar Yuyuan Liu

delete temp files

parent 92c685fd
Pipeline #269202 failed with stage
in 53 minutes and 42 seconds
train.log
target/
/.idea
This diff is collapsed.
Manifest-Version: 1.0
Archiver-Version: Plexus Archiver
Created-By: 25.242-b08 (Private Build)
FROM registry.git.rwth-aachen.de/monticore/embeddedmontiarc/generators/emadl2cpp/integrationtests/mxnet:v0.0.3
RUN pip install opencv-python
# Install Open CV - Warning, this takes absolutely forever
RUN apt-get update && apt-get install -y libgdal-dev
RUN mkdir -p ~/opencv cd ~/opencv && \
wget https://github.com/Itseez/opencv/archive/3.0.0.zip && \
unzip 3.0.0.zip && \
rm 3.0.0.zip && \
mv opencv-3.0.0 OpenCV
RUN apt-get install -y qtbase5-dev
RUN apt-get install -y build-essential
ENV PATH="${PATH}:/usr/include/"
RUN cd OpenCV && \
mkdir build && \
cd build && \
cmake \
-DWITH_QT=ON \
-DWITH_OPENGL=ON \
-DFORCE_VTK=ON \
-DWITH_TBB=ON \
-DWITH_GDAL=ON \
-DWITH_XINE=ON \
-DBUILD_EXAMPLES=ON \
-D ENABLE_PRECOMPILED_HEADERS=OFF ..
RUN cd OpenCV/build && make -j4
RUN cd OpenCV/build && make install
RUN cd OpenCV/build && ldconfig
RUN apt-get install -y libopenblas-dev
# (c) https://github.com/MontiCore/monticore
from __future__ import print_function
from __future__ import division
import h5py
import numpy as np
import cv2
import os
import argparse
import errno
import random
import sys
def create_img_list(name, data_path):
dir_name = data_path + "/" + name
image_paths = []
image_class_indices = []
print(dir_name)
for class_index_name in os.listdir(dir_name):
class_dir_path = dir_name + "/" + class_index_name
if os.path.isdir(class_dir_path):
for image_name in os.listdir(class_dir_path):
image_path = class_dir_path + "/" + image_name
image_paths.append(image_path)
class_index = float(class_index_name)
image_class_indices.append(class_index)
return image_paths, image_class_indices
def create_h5_from_list(image_paths, image_class_indices, target_dir, target_file_name, input_port_name, output_port_name, shuffle=True):
img = cv2.imread(image_paths[0])
t_img = np.transpose(img, (2,0,1)).astype(np.float32)
#t_img = t_img[-1:,:,:]
#print(t_img)
channels = t_img.shape[0]
height = t_img.shape[1]
width = t_img.shape[2]
data_size = len(image_paths)
target_file = target_dir + "/" + target_file_name + ".h5"
if os.path.isfile(target_file):
print("File", target_file, "already exists. Skipping data file creation.")
return
try:
os.makedirs(target_dir)
except OSError as e:
if e.errno != errno.EEXIST:
raise
if shuffle:
combined = list(zip(image_paths, image_class_indices))
random.shuffle(combined)
image_paths[:], image_class_indices[:] = zip(*combined)
print("Creating " + target_file + " (images:" + str(data_size) + ", channels:" + str(channels) + ", height:" + str(height) + ", width:" + str(width) + "):")
with h5py.File(target_file, "w") as ofile:
in_dset = ofile.create_dataset(input_port_name, (data_size,channels, height, width), dtype=np.float32)
out_dset = ofile.create_dataset(output_port_name + "_label", (data_size,), dtype=np.float32)
for i in range(data_size):
img = cv2.imread(image_paths[i])
t_img = np.transpose(img, (2,0,1)).astype(np.float32)
#t_img = t_img[-1:,:,:]
in_dset[i] = t_img
out_dset[i] = image_class_indices[i]
#print progress
if i % 100 == 0:
percentage = 100*i / data_size
sys.stdout.write("\r{:0.1f}%".format(percentage))
sys.stdout.flush()
sys.stdout.write("\r100.0%\n")
sys.stdout.flush()
if __name__ == "__main__":
parser = argparse.ArgumentParser(description='Translate image directories into hdf5 training sets for EMADL.')
parser.add_argument("--in_port", action="store", dest="in_port", default="data")
parser.add_argument("--out_port", action="store", dest="out_port", default="softmax")
parser.add_argument("--data_path", action="store", dest="data_path", default=".")
parser.add_argument("--target_path", action="store", dest="target_path", default=".")
args = parser.parse_args()
for file_name in os.listdir(args.data_path):
if file_name == "train":
image_paths, image_class_indices = create_img_list(file_name, args.data_path)
create_h5_from_list(image_paths, image_class_indices, args.target_path, file_name, args.in_port, args.out_port)
if file_name == "test":
image_paths, image_class_indices = create_img_list(file_name, args.data_path)
create_h5_from_list(image_paths, image_class_indices, args.target_path, file_name, args.in_port, args.out_port)
Manifest-Version: 1.0
Archiver-Version: Plexus Archiver
Created-By: 25.242-b08 (Private Build)
#Generated by Maven
#Mon Apr 13 22:19:30 CEST 2020
version=0.0.1-SNAPSHOT
groupId=de.monticore.lang.monticar.application
artifactId=mnist-calculator
cmake_minimum_required(VERSION 3.5)
set(CMAKE_CXX_STANDARD 14)
project(cNNCalculator_connector LANGUAGES CXX)
#set cmake module path
set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${CMAKE_CURRENT_SOURCE_DIR}/cmake)
# add dependencies
find_package(Armadillo REQUIRED)
set(INCLUDE_DIRS ${INCLUDE_DIRS} ${Armadillo_INCLUDE_DIRS})
set(LIBS ${LIBS} ${Armadillo_LIBRARIES})
# additional commands
set(LIBS ${LIBS} mxnet)
# create static library
include_directories(${INCLUDE_DIRS})
add_library(cNNCalculator_connector cNNCalculator_connector.cpp)
target_include_directories(cNNCalculator_connector PUBLIC ${CMAKE_CURRENT_SOURCE_DIR} ${INCLUDE_DIRS})
target_link_libraries(cNNCalculator_connector PUBLIC ${LIBS})
set_target_properties(cNNCalculator_connector PROPERTIES LINKER_LANGUAGE CXX)
# export cmake project
export(TARGETS cNNCalculator_connector FILE cNNCalculator_connector.cmake)
# additional commands end
#ifndef CNNBUFFERFILE_H
#define CNNBUFFERFILE_H
#include <stdio.h>
#include <iostream>
#include <fstream>
// Read file to buffer
class BufferFile {
public :
std::string file_path_;
int length_;
char* buffer_;
explicit BufferFile(std::string file_path)
:file_path_(file_path) {
std::ifstream ifs(file_path.c_str(), std::ios::in | std::ios::binary);
if (!ifs) {
std::cerr << "Can't open the file. Please check " << file_path << ". \n";
length_ = 0;
buffer_ = NULL;
return;
}
ifs.seekg(0, std::ios::end);
length_ = ifs.tellg();
ifs.seekg(0, std::ios::beg);
std::cout << file_path.c_str() << " ... "<< length_ << " bytes\n";
buffer_ = new char[sizeof(char) * length_];
ifs.read(buffer_, length_);
ifs.close();
}
int GetLength() {
return length_;
}
char* GetBuffer() {
return buffer_;
}
~BufferFile() {
if (buffer_) {
delete[] buffer_;
buffer_ = NULL;
}
}
};
#endif // CNNBUFFERFILE_H
import mxnet as mx
import logging
import os
from CNNNet_cNNCalculator_connector_predictor1 import Net_0
class CNNCreator_cNNCalculator_connector_predictor1:
_model_dir_ = "model/cNNCalculator.Network/"
_model_prefix_ = "model"
def __init__(self):
self.weight_initializer = mx.init.Normal()
self.networks = {}
def load(self, context):
earliestLastEpoch = None
for i, network in self.networks.items():
lastEpoch = 0
param_file = None
try:
os.remove(self._model_dir_ + self._model_prefix_ + "_" + str(i) + "_newest-0000.params")
except OSError:
pass
try:
os.remove(self._model_dir_ + self._model_prefix_ + "_" + str(i) + "_newest-symbol.json")
except OSError:
pass
if os.path.isdir(self._model_dir_):
for file in os.listdir(self._model_dir_):
if ".params" in file and self._model_prefix_ + "_" + str(i) in file:
epochStr = file.replace(".params","").replace(self._model_prefix_ + "_" + str(i) + "-","")
epoch = int(epochStr)
if epoch > lastEpoch:
lastEpoch = epoch
param_file = file
if param_file is None:
earliestLastEpoch = 0
else:
logging.info("Loading checkpoint: " + param_file)
network.load_parameters(self._model_dir_ + param_file)
if earliestLastEpoch == None or lastEpoch < earliestLastEpoch:
earliestLastEpoch = lastEpoch
return earliestLastEpoch
def construct(self, context, data_mean=None, data_std=None):
self.networks[0] = Net_0(data_mean=data_mean, data_std=data_std)
self.networks[0].collect_params().initialize(self.weight_initializer, ctx=context)
self.networks[0].hybridize()
self.networks[0](mx.nd.zeros((1, 1,28,28,), ctx=context))
if not os.path.exists(self._model_dir_):
os.makedirs(self._model_dir_)
for i, network in self.networks.items():
network.export(self._model_dir_ + self._model_prefix_ + "_" + str(i), epoch=0)
import mxnet as mx
import logging
import os
from CNNNet_cNNCalculator_connector_predictor2 import Net_0
class CNNCreator_cNNCalculator_connector_predictor2:
_model_dir_ = "model/cNNCalculator.Network/"
_model_prefix_ = "model"
def __init__(self):
self.weight_initializer = mx.init.Normal()
self.networks = {}
def load(self, context):
earliestLastEpoch = None
for i, network in self.networks.items():
lastEpoch = 0
param_file = None
try:
os.remove(self._model_dir_ + self._model_prefix_ + "_" + str(i) + "_newest-0000.params")
except OSError:
pass
try:
os.remove(self._model_dir_ + self._model_prefix_ + "_" + str(i) + "_newest-symbol.json")
except OSError:
pass
if os.path.isdir(self._model_dir_):
for file in os.listdir(self._model_dir_):
if ".params" in file and self._model_prefix_ + "_" + str(i) in file:
epochStr = file.replace(".params","").replace(self._model_prefix_ + "_" + str(i) + "-","")
epoch = int(epochStr)
if epoch > lastEpoch:
lastEpoch = epoch
param_file = file
if param_file is None:
earliestLastEpoch = 0
else:
logging.info("Loading checkpoint: " + param_file)
network.load_parameters(self._model_dir_ + param_file)
if earliestLastEpoch == None or lastEpoch < earliestLastEpoch:
earliestLastEpoch = lastEpoch
return earliestLastEpoch
def construct(self, context, data_mean=None, data_std=None):
self.networks[0] = Net_0(data_mean=data_mean, data_std=data_std)
self.networks[0].collect_params().initialize(self.weight_initializer, ctx=context)
self.networks[0].hybridize()
self.networks[0](mx.nd.zeros((1, 1,28,28,), ctx=context))
if not os.path.exists(self._model_dir_):
os.makedirs(self._model_dir_)
for i, network in self.networks.items():
network.export(self._model_dir_ + self._model_prefix_ + "_" + str(i), epoch=0)
import mxnet as mx
import logging
import os
from CNNNet_cNNCalculator_connector_predictor3 import Net_0
class CNNCreator_cNNCalculator_connector_predictor3:
_model_dir_ = "model/cNNCalculator.Network/"
_model_prefix_ = "model"
def __init__(self):
self.weight_initializer = mx.init.Normal()
self.networks = {}
def load(self, context):
earliestLastEpoch = None
for i, network in self.networks.items():
lastEpoch = 0
param_file = None
try:
os.remove(self._model_dir_ + self._model_prefix_ + "_" + str(i) + "_newest-0000.params")
except OSError:
pass
try:
os.remove(self._model_dir_ + self._model_prefix_ + "_" + str(i) + "_newest-symbol.json")
except OSError:
pass
if os.path.isdir(self._model_dir_):
for file in os.listdir(self._model_dir_):
if ".params" in file and self._model_prefix_ + "_" + str(i) in file:
epochStr = file.replace(".params","").replace(self._model_prefix_ + "_" + str(i) + "-","")
epoch = int(epochStr)
if epoch > lastEpoch:
lastEpoch = epoch
param_file = file
if param_file is None:
earliestLastEpoch = 0
else:
logging.info("Loading checkpoint: " + param_file)
network.load_parameters(self._model_dir_ + param_file)
if earliestLastEpoch == None or lastEpoch < earliestLastEpoch:
earliestLastEpoch = lastEpoch
return earliestLastEpoch
def construct(self, context, data_mean=None, data_std=None):
self.networks[0] = Net_0(data_mean=data_mean, data_std=data_std)
self.networks[0].collect_params().initialize(self.weight_initializer, ctx=context)
self.networks[0].hybridize()
self.networks[0](mx.nd.zeros((1, 1,28,28,), ctx=context))
if not os.path.exists(self._model_dir_):
os.makedirs(self._model_dir_)
for i, network in self.networks.items():
network.export(self._model_dir_ + self._model_prefix_ + "_" + str(i), epoch=0)
import mxnet as mx
import logging
import os
from CNNNet_cNNCalculator_connector_predictor4 import Net_0
class CNNCreator_cNNCalculator_connector_predictor4:
_model_dir_ = "model/cNNCalculator.Network/"
_model_prefix_ = "model"
def __init__(self):
self.weight_initializer = mx.init.Normal()
self.networks = {}
def load(self, context):
earliestLastEpoch = None
for i, network in self.networks.items():
lastEpoch = 0
param_file = None
try:
os.remove(self._model_dir_ + self._model_prefix_ + "_" + str(i) + "_newest-0000.params")
except OSError:
pass
try:
os.remove(self._model_dir_ + self._model_prefix_ + "_" + str(i) + "_newest-symbol.json")
except OSError:
pass
if os.path.isdir(self._model_dir_):
for file in os.listdir(self._model_dir_):
if ".params" in file and self._model_prefix_ + "_" + str(i) in file:
epochStr = file.replace(".params","").replace(self._model_prefix_ + "_" + str(i) + "-","")
epoch = int(epochStr)
if epoch > lastEpoch:
lastEpoch = epoch
param_file = file
if param_file is None:
earliestLastEpoch = 0
else:
logging.info("Loading checkpoint: " + param_file)
network.load_parameters(self._model_dir_ + param_file)
if earliestLastEpoch == None or lastEpoch < earliestLastEpoch:
earliestLastEpoch = lastEpoch
return earliestLastEpoch
def construct(self, context, data_mean=None, data_std=None):
self.networks[0] = Net_0(data_mean=data_mean, data_std=data_std)
self.networks[0].collect_params().initialize(self.weight_initializer, ctx=context)
self.networks[0].hybridize()
self.networks[0](mx.nd.zeros((1, 1,28,28,), ctx=context))
if not os.path.exists(self._model_dir_):
os.makedirs(self._model_dir_)
for i, network in self.networks.items():