Commit 38974c6a authored by Julian Johannes Steinsberger-Dührßen's avatar Julian Johannes Steinsberger-Dührßen
Browse files

Test fix, Updated Readme

parent c32ee51e
Pipeline #326395 passed with stage
in 2 minutes and 59 seconds
......@@ -30,9 +30,9 @@ See example project [EMADL-Demo](https://git.rwth-aachen.de/thomas.timmermanns/E
* prediction - generated code is C++.
## Installation
A new bash script for mxnet/gluon can be found [installation scripts](https://git.rwth-aachen.de/monticore/EmbeddedMontiArc/generators/EMADL2CPP/tree/tensorflow_group/src/main/resources/installation_scripts)
A new bash script for mxnet/gluon can be found [installation scripts](https://git.rwth-aachen.de/monticore/EmbeddedMontiArc/generators/EMADL2CPP/-/tree/master/src/main/resources/installation_scripts)
changing the installation process for mxnet for cpp. This fill now install the full cpp api and not the reduced c api. This script will install all dependencies both for python and cpp as of 26.08.2020.
Additionally a simmilar docker script used for the git ci pipeline can be found in the gluon subfolder at [Docker images](https://git.rwth-aachen.de/monticore/EmbeddedMontiArc/generators/EMADL2CPP/tree/tensorflow_group/src/test/resources/docker).
Additionally a similar docker script used for the git ci pipeline can be found in the gluon subfolder at [Docker images](https://git.rwth-aachen.de/monticore/EmbeddedMontiArc/generators/EMADL2CPP/-/tree/master/src/test/resources/docker).
The other two bash scripts found in the installation_scripts folder are outdated but may be consulted for installation guidlines for other backends.
Note that the installation may take some time (hours) enough RAM or a big enough swapspace is advisable (>10GB). This scripts were tested with a completly clean Ubuntu 16.04,
without system updates installed. Using another Ubuntu version or installing other stuff, system updates included might/ have caused problems.
......
......@@ -42,7 +42,7 @@ public class GenerationTest extends AbstractSymtabTest {
"cifar10_cifar10Classifier.cpp",
"cifar10_cifar10Classifier.h",
"CNNCreator_cifar10_cifar10Classifier_net.py",
"CNNBufferFile.h",
"CNNModelLoader.h",
"CNNPredictor_cifar10_cifar10Classifier_net.h",
"cifar10_cifar10Classifier_net.h",
"CNNTranslator.h",
......@@ -182,7 +182,7 @@ public class GenerationTest extends AbstractSymtabTest {
Paths.get("./target/generated-sources-emadl"),
Paths.get("./src/test/resources/target_code/gluon"),
Arrays.asList(
"CNNBufferFile.h",
"CNNModelLoader.h",
"CNNNet_mnist_mnistClassifier_net.py",
"mnist_mnistClassifier.cpp",
"mnist_mnistClassifier.h",
......@@ -221,7 +221,7 @@ public class GenerationTest extends AbstractSymtabTest {
"cartpole_master_dqn.h",
"cartpole_master_policy.h",
"CMakeLists.txt",
"CNNBufferFile.h",
"CNNModelLoader.h",
"CNNCreator_cartpole_master_dqn.py",
"CNNNet_cartpole_master_dqn.py",
"CNNPredictor_cartpole_master_dqn.h",
......@@ -266,7 +266,7 @@ public class GenerationTest extends AbstractSymtabTest {
"mountaincar_master.h",
"mountaincar_master_actor.h",
"CMakeLists.txt",
"CNNBufferFile.h",
"CNNModelLoader.h",
"CNNCreator_mountaincar_master_actor.py",
"CNNNet_mountaincar_master_actor.py",
"CNNPredictor_mountaincar_master_actor.h",
......
......@@ -27,7 +27,7 @@ public class IntegrationPythonWrapperTest extends AbstractSymtabTest {
Paths.get("./src/test/resources/target_code/gluon/reinforcementModel/torcs"),
Arrays.asList(
"CMakeLists.txt",
"CNNBufferFile.h",
"CNNModelLoader.h",
"torcs_agent_torcsAgent.cpp",
"torcs_agent_torcsAgent.h",
"torcs_agent_torcsAgent_dqn.h",
......@@ -82,7 +82,7 @@ public class IntegrationPythonWrapperTest extends AbstractSymtabTest {
Paths.get("./src/test/resources/target_code/gluon/reinforcementModel/torcs_td3"),
Arrays.asList(
"CMakeLists.txt",
"CNNBufferFile.h",
"CNNModelLoader.h",
"torcs_agent_torcsAgent.cpp",
"torcs_agent_torcsAgent.h",
"torcs_agent_torcsAgent_actor.h",
......
#ifndef CNNBUFFERFILE_H
#define CNNBUFFERFILE_H
#include <stdio.h>
#include <iostream>
#include <fstream>
// Read file to buffer
class BufferFile {
public :
std::string file_path_;
int length_;
char* buffer_;
explicit BufferFile(std::string file_path)
:file_path_(file_path) {
std::ifstream ifs(file_path.c_str(), std::ios::in | std::ios::binary);
if (!ifs) {
std::cerr << "Can't open the file. Please check " << file_path << ". \n";
length_ = 0;
buffer_ = NULL;
return;
}
ifs.seekg(0, std::ios::end);
length_ = ifs.tellg();
ifs.seekg(0, std::ios::beg);
std::cout << file_path.c_str() << " ... "<< length_ << " bytes\n";
buffer_ = new char[sizeof(char) * length_];
ifs.read(buffer_, length_);
ifs.close();
}
int GetLength() {
return length_;
}
char* GetBuffer() {
return buffer_;
}
~BufferFile() {
if (buffer_) {
delete[] buffer_;
buffer_ = NULL;
}
}
};
#endif // CNNBUFFERFILE_H
#ifndef CNNMODELLOADER
#define CNNMODELLOADER
#include <mxnet-cpp/MxNetCpp.h>
#include <stdio.h>
#include <iostream>
#include <fstream>
using namespace mxnet::cpp;
// Read files to load moddel symbol and parameters
class ModelLoader {
private:
Context ctx = Context::cpu();
std::vector<Symbol> network_symbol_list;
std::vector<std::map<std::string, NDArray>> network_param_map_list;
std::vector<Symbol> query_symbol_list;
std::vector<std::map<std::string, NDArray>> query_param_map_list;
std::vector<std::map<std::string, NDArray>> replay_memory;
std::vector<Symbol> loss_symbol;
std::vector<std::map<std::string, NDArray>> loss_param_map;
void checkFile(std::string file_path){
std::ifstream ifs(file_path.c_str(), std::ios::in | std::ios::binary);
if (!ifs) {
std::cerr << "Can't open the file. Please check " << file_path << ". \n";
return;
}
int length_;
ifs.seekg(0, std::ios::end);
length_ = ifs.tellg();
ifs.seekg(0, std::ios::beg);
std::cout << file_path.c_str() << " ... "<< length_ << " bytes\n";
ifs.close();
}
void loadComponent(std::string json_path,
std::string param_path,
std::vector<Symbol> &symbols_list,
std::vector<std::map<std::string, NDArray>> &param_map_list){
checkFile(json_path);
symbols_list.push_back(Symbol::Load(json_path));
checkFile(param_path);
std::map<std::string, NDArray> params;
NDArray::Load(param_path, 0, &params);
param_map_list.push_back(processParamMap(params));
}
std::map<std::string, NDArray> processParamMap(std::map<std::string, NDArray> param_map){
std::map<std::string, NDArray> processed_param_map;
if(!param_map.empty()){
for (const auto &pair : param_map) {
std::string name = pair.first.substr(4); //the first four letters would be the type (arg: or aux:, but we don't have aux parameters? <- need to make sure)
processed_param_map[name] = pair.second.Copy(ctx);
}
}
return processed_param_map;
}
public:
explicit ModelLoader(std::string file_prefix, mx_uint num_subnets, Context ctx_param){
ctx = ctx_param;
std::string network_json_path;
std::string network_param_path;
std::string query_json_path;
std::string query_param_path;
std::string memory_path;
std::string loss_json_path;
std::string loss_param_path;
//Load network
if(!num_subnets){
network_json_path = file_prefix + "-symbol.json";
network_param_path = file_prefix + "-0000.params";
loadComponent(network_json_path, network_param_path, network_symbol_list, network_param_map_list);
}else{
for(int i=0; i < num_subnets; i++){
network_json_path = file_prefix + "_episodic_sub_net_" + std::to_string(i) + "-symbol.json";
network_param_path = file_prefix + "_episodic_sub_net_" + std::to_string(i) + "-0000.params";
loadComponent(network_json_path, network_param_path, network_symbol_list, network_param_map_list);
if(i >= 1){
query_json_path = file_prefix + "_episodic_query_net_" + std::to_string(i) + "-symbol.json";
query_param_path = file_prefix + "_episodic_query_net_" + std::to_string(i) + "-0000.params";
loadComponent(query_json_path, query_param_path, query_symbol_list, query_param_map_list);
memory_path = file_prefix + "_episodic_memory_sub_net_" + std::to_string(i) + "-0000";
checkFile(memory_path);
std::map<std::string, NDArray> mem_map = NDArray::LoadToMap(memory_path);
for(auto &mem : mem_map){
mem.second = mem.second.Copy(ctx);
}
replay_memory.push_back(mem_map);
}
}
}
//Load Loss
loss_json_path = file_prefix + "_loss-symbol.json";
loss_param_path = file_prefix + "_loss-0000.params";
loadComponent(loss_json_path, loss_param_path, loss_symbol, loss_param_map);
NDArray::WaitAll();
}
std::vector<Symbol> GetNetworkSymbols() {
return network_symbol_list;
}
std::vector<std::map<std::string, NDArray>> GetNetworkParamMaps() {
return network_param_map_list;
}
Symbol GetLoss() {
return loss_symbol[0];
}
std::map<std::string, NDArray> GetLossParamMap() {
return loss_param_map[0];
}
std::vector<Symbol> GetQuerySymbols() {
return query_symbol_list;
}
std::vector<std::map<std::string, NDArray>> GetQueryParamMaps() {
return query_param_map_list;
}
std::vector<std::map<std::string, NDArray>> GetReplayMemory(){
return replay_memory;
}
};
#endif // CNNMODELLOADER
#ifndef CNNBUFFERFILE_H
#define CNNBUFFERFILE_H
#include <stdio.h>
#include <iostream>
#include <fstream>
// Read file to buffer
class BufferFile {
public :
std::string file_path_;
int length_;
char* buffer_;
explicit BufferFile(std::string file_path)
:file_path_(file_path) {
std::ifstream ifs(file_path.c_str(), std::ios::in | std::ios::binary);
if (!ifs) {
std::cerr << "Can't open the file. Please check " << file_path << ". \n";
length_ = 0;
buffer_ = NULL;
return;
}
ifs.seekg(0, std::ios::end);
length_ = ifs.tellg();
ifs.seekg(0, std::ios::beg);
std::cout << file_path.c_str() << " ... "<< length_ << " bytes\n";
buffer_ = new char[sizeof(char) * length_];
ifs.read(buffer_, length_);
ifs.close();
}
int GetLength() {
return length_;
}
char* GetBuffer() {
return buffer_;
}
~BufferFile() {
if (buffer_) {
delete[] buffer_;
buffer_ = NULL;
}
}
};
#endif // CNNBUFFERFILE_H
#ifndef CNNMODELLOADER
#define CNNMODELLOADER
#include <mxnet-cpp/MxNetCpp.h>
#include <stdio.h>
#include <iostream>
#include <fstream>
using namespace mxnet::cpp;
// Read files to load moddel symbol and parameters
class ModelLoader {
private:
Context ctx = Context::cpu();
std::vector<Symbol> network_symbol_list;
std::vector<std::map<std::string, NDArray>> network_param_map_list;
std::vector<Symbol> query_symbol_list;
std::vector<std::map<std::string, NDArray>> query_param_map_list;
std::vector<std::map<std::string, NDArray>> replay_memory;
std::vector<Symbol> loss_symbol;
std::vector<std::map<std::string, NDArray>> loss_param_map;
void checkFile(std::string file_path){
std::ifstream ifs(file_path.c_str(), std::ios::in | std::ios::binary);
if (!ifs) {
std::cerr << "Can't open the file. Please check " << file_path << ". \n";
return;
}
int length_;
ifs.seekg(0, std::ios::end);
length_ = ifs.tellg();
ifs.seekg(0, std::ios::beg);
std::cout << file_path.c_str() << " ... "<< length_ << " bytes\n";
ifs.close();
}
void loadComponent(std::string json_path,
std::string param_path,
std::vector<Symbol> &symbols_list,
std::vector<std::map<std::string, NDArray>> &param_map_list){
checkFile(json_path);
symbols_list.push_back(Symbol::Load(json_path));
checkFile(param_path);
std::map<std::string, NDArray> params;
NDArray::Load(param_path, 0, &params);
param_map_list.push_back(processParamMap(params));
}
std::map<std::string, NDArray> processParamMap(std::map<std::string, NDArray> param_map){
std::map<std::string, NDArray> processed_param_map;
if(!param_map.empty()){
for (const auto &pair : param_map) {
std::string name = pair.first.substr(4); //the first four letters would be the type (arg: or aux:, but we don't have aux parameters? <- need to make sure)
processed_param_map[name] = pair.second.Copy(ctx);
}
}
return processed_param_map;
}
public:
explicit ModelLoader(std::string file_prefix, mx_uint num_subnets, Context ctx_param){
ctx = ctx_param;
std::string network_json_path;
std::string network_param_path;
std::string query_json_path;
std::string query_param_path;
std::string memory_path;
std::string loss_json_path;
std::string loss_param_path;
//Load network
if(!num_subnets){
network_json_path = file_prefix + "-symbol.json";
network_param_path = file_prefix + "-0000.params";
loadComponent(network_json_path, network_param_path, network_symbol_list, network_param_map_list);
}else{
for(int i=0; i < num_subnets; i++){
network_json_path = file_prefix + "_episodic_sub_net_" + std::to_string(i) + "-symbol.json";
network_param_path = file_prefix + "_episodic_sub_net_" + std::to_string(i) + "-0000.params";
loadComponent(network_json_path, network_param_path, network_symbol_list, network_param_map_list);
if(i >= 1){
query_json_path = file_prefix + "_episodic_query_net_" + std::to_string(i) + "-symbol.json";
query_param_path = file_prefix + "_episodic_query_net_" + std::to_string(i) + "-0000.params";
loadComponent(query_json_path, query_param_path, query_symbol_list, query_param_map_list);
memory_path = file_prefix + "_episodic_memory_sub_net_" + std::to_string(i) + "-0000";
checkFile(memory_path);
std::map<std::string, NDArray> mem_map = NDArray::LoadToMap(memory_path);
for(auto &mem : mem_map){
mem.second = mem.second.Copy(ctx);
}
replay_memory.push_back(mem_map);
}
}
}
//Load Loss
loss_json_path = file_prefix + "_loss-symbol.json";
loss_param_path = file_prefix + "_loss-0000.params";
loadComponent(loss_json_path, loss_param_path, loss_symbol, loss_param_map);
NDArray::WaitAll();
}
std::vector<Symbol> GetNetworkSymbols() {
return network_symbol_list;
}
std::vector<std::map<std::string, NDArray>> GetNetworkParamMaps() {
return network_param_map_list;
}
Symbol GetLoss() {
return loss_symbol[0];
}
std::map<std::string, NDArray> GetLossParamMap() {
return loss_param_map[0];
}
std::vector<Symbol> GetQuerySymbols() {
return query_symbol_list;
}
std::vector<std::map<std::string, NDArray>> GetQueryParamMaps() {
return query_param_map_list;
}
std::vector<std::map<std::string, NDArray>> GetReplayMemory(){
return replay_memory;
}
};
#endif // CNNMODELLOADER
#ifndef CNNBUFFERFILE_H
#define CNNBUFFERFILE_H
#include <stdio.h>
#include <iostream>
#include <fstream>
// Read file to buffer
class BufferFile {
public :
std::string file_path_;
int length_;
char* buffer_;
explicit BufferFile(std::string file_path)
:file_path_(file_path) {
std::ifstream ifs(file_path.c_str(), std::ios::in | std::ios::binary);
if (!ifs) {
std::cerr << "Can't open the file. Please check " << file_path << ". \n";
length_ = 0;
buffer_ = NULL;
return;
}
ifs.seekg(0, std::ios::end);
length_ = ifs.tellg();
ifs.seekg(0, std::ios::beg);
std::cout << file_path.c_str() << " ... "<< length_ << " bytes\n";
buffer_ = new char[sizeof(char) * length_];
ifs.read(buffer_, length_);
ifs.close();
}
int GetLength() {
return length_;
}
char* GetBuffer() {
return buffer_;
}
~BufferFile() {
if (buffer_) {
delete[] buffer_;
buffer_ = NULL;
}
}
};
#endif // CNNBUFFERFILE_H
#ifndef CNNMODELLOADER
#define CNNMODELLOADER
#include <mxnet-cpp/MxNetCpp.h>
#include <stdio.h>
#include <iostream>
#include <fstream>
using namespace mxnet::cpp;
// Read files to load moddel symbol and parameters
class ModelLoader {
private:
Context ctx = Context::cpu();
std::vector<Symbol> network_symbol_list;
std::vector<std::map<std::string, NDArray>> network_param_map_list;
std::vector<Symbol> query_symbol_list;
std::vector<std::map<std::string, NDArray>> query_param_map_list;
std::vector<std::map<std::string, NDArray>> replay_memory;
std::vector<Symbol> loss_symbol;
std::vector<std::map<std::string, NDArray>> loss_param_map;
void checkFile(std::string file_path){
std::ifstream ifs(file_path.c_str(), std::ios::in | std::ios::binary);
if (!ifs) {
std::cerr << "Can't open the file. Please check " << file_path << ". \n";
return;
}
int length_;
ifs.seekg(0, std::ios::end);
length_ = ifs.tellg();
ifs.seekg(0, std::ios::beg);
std::cout << file_path.c_str() << " ... "<< length_ << " bytes\n";
ifs.close();
}
void loadComponent(std::string json_path,
std::string param_path,
std::vector<Symbol> &symbols_list,
std::vector<std::map<std::string, NDArray>> &param_map_list){
checkFile(json_path);
symbols_list.push_back(Symbol::Load(json_path));
checkFile(param_path);
std::map<std::string, NDArray> params;
NDArray::Load(param_path, 0, &params);
param_map_list.push_back(processParamMap(params));
}
std::map<std::string, NDArray> processParamMap(std::map<std::string, NDArray> param_map){
std::map<std::string, NDArray> processed_param_map;
if(!param_map.empty()){
for (const auto &pair : param_map) {
std::string name = pair.first.substr(4); //the first four letters would be the type (arg: or aux:, but we don't have aux parameters? <- need to make sure)
processed_param_map[name] = pair.second.Copy(ctx);
}
}
return processed_param_map;
}
public:
explicit ModelLoader(std::string file_prefix, mx_uint num_subnets, Context ctx_param){
ctx = ctx_param;
std::string network_json_path;