Skip to content

GitLab

Commit a9103600 authored by Svetlana's avatar Svetlana
Browse files

Merge branch 'develop' of git.rwth-aachen.de:autonomousdriving/torcs_dl into develop

parents dc46cbcf eeda3474
Hide whitespace changes
Inline Side-by-side
CNNModels/.gitignore
View file @ a9103600
target/
output/
generated/
out/
.idea/
.git
*.iml
\ No newline at end of file
*.iml
CNNModels/generate.sh 0 → 100755
View file @ a9103600
#!/usr/bin/env bash
echo "Generating files.."
java -jar embedded-montiarc-emadl-generator-0.2.1-SNAPSHOT-jar-with-dependencies.jar -m src/models -r Dpnet -o generated
CNNModels/pom.xml deleted 100644 → 0
View file @ dc46cbcf
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>de.monticore.lang.monticar</groupId>
<artifactId>cnn-model-training</artifactId>
<version>1.0-SNAPSHOT</version>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.3</version>
<configuration>
<useIncrementalCompilation>true</useIncrementalCompilation>
<source>1.8</source>
<target>1.8</target>
</configuration>
</plugin>
</plugins>
</build>
<dependencies>
<dependency>
<groupId>de.monticore.lang.monticar</groupId>
<artifactId>embedded-montiarc-emadl-generator</artifactId>
<version>0.2.1-SNAPSHOT</version>
</dependency>
</dependencies>
</project>
CNNModels/src/main/java/de/monticore/lang/monticar/torcs_dl/CNNCodeGenerator.java deleted 100644 → 0
View file @ dc46cbcf
package de.monticore.lang.monticar.torcs_dl;
import de.monticore.lang.monticar.emadl.generator.EMADLGenerator;
import freemarker.template.TemplateException;
import java.io.IOException;
import java.nio.file.Path;
import java.nio.file.Paths;
public class CNNCodeGenerator {
public static final String TARGET_PATH_GENERATED = "./target/generated-sources-cnn-model-training/";
public static final String MODELS_PATH = "src/main/models/";
public static void main(String[] args) {
System.out.println("Starting code generation...");
EMADLGenerator gen = new EMADLGenerator();
gen.setGenerationTargetPath(TARGET_PATH_GENERATED);
try {
gen.generate(MODELS_PATH, "Dpnet");
} catch (IOException | TemplateException e) {
e.printStackTrace();
}
}
}
CNNModels/src/main/models/Dpnet.cnnt → CNNModels/src/models/Dpnet.cnnt
View file @ a9103600
......@@ -2,10 +2,13 @@ configuration Dpnet{
num_epoch : 100
batch_size : 64
context:cpu
normalize: false
normalize: true
optimizer : sgd{
learning_rate : 0.01
weight_decay : 0.0005
// reduce the learning rate starting from 0.01 every 8000 iterations by a factor of 0.9 (decrease by 10%)
learning_rate_decay: 0.9
step_size: 8000
learning_rate_minimum: 0.01
}
}
DataPreparation/torcs_data_preparation.py
View file @ a9103600
......@@ -4,6 +4,7 @@ import csv
import cv2
import datetime
import h5py
import matplotlib.pyplot as plt
import mxnet as mx
import numpy as np
from PIL import Image
......@@ -12,6 +13,9 @@ from sklearn.cross_validation import train_test_split
import tarfile
import os
TEST_REC = "torcs_test.rec"
TRAIN_REC = "torcs_train.rec"
ARCHIVE = False
CHUNK_SIZE = 10000
LEVELDB_PATH = "/media/sveta/4991e634-dd81-4cb9-bf46-2fa9c7159263/TORCS_Training_1F"
......@@ -19,34 +23,63 @@ HDF5_PATH = "/media/sveta/4991e634-dd81-4cb9-bf46-2fa9c7159263/TORCS_HDF5_3/"
RAW_PATH = "/media/sveta/4991e634-dd81-4cb9-bf46-2fa9c7159263/TORCS_raw/"
EXAMPLES_PATH = "/media/sveta/4991e634-dd81-4cb9-bf46-2fa9c7159263/TORCS_examples/"
TRAIN_MEAN = [99.39394537, 110.60877108, 117.86127587]
TRAIN_STD = [42.04910545, 49.47874084, 62.61726178]
def main():
start_date = datetime.datetime.now()
# leveldb_to_rec(start_date)
read_from_recordio()
# read_from_recordio()
# compute_train_mean()
test_normalization()
def compute_train_mean():
record = mx.recordio.MXRecordIO(RAW_PATH + "torcs_train.rec", "r")
img_stats = []
for i in range(50):
record = mx.recordio.MXRecordIO(RAW_PATH + TRAIN_REC, "r")
all_means = list()
all_std = list()
for i in range(387851):
if i % 1000 == 0:
print(i)
item = record.read()
header, img_as_array = mx.recordio.unpack_img(item)
# img is RGB of shape (210, 280, 3)
# img = Image.fromarray(img_as_array)
# img.show()
mean, std = cv2.meanStdDev(img_as_array)
mean_as_img = Image.fromarray(mean)
mean_as_img.show()
img_stats.append(np.array([mean[::-1] / 255, std[::-1] / 255]))
img_stats = np.mean(img_stats, axis=0)
print(img_stats)
all_means.append(mean)
all_std.append(std)
# img_stats.append(np.array([mean[::-1] / 255, std[::-1] / 255]))
mean = np.mean(all_means, axis=0)
print("MEAN")
print(mean)
std = np.mean(all_std, axis=0)
print("STD")
print(std)
def test_normalization():
width = 280
height = 210
data_mean = np.asarray([[[a] * width] * height for a in TRAIN_MEAN]) # (3, 280, 210)
data_std = np.asarray([[[a] * width] * height for a in TRAIN_STD])
record = mx.recordio.MXRecordIO(RAW_PATH + TRAIN_REC, "r")
item = record.read()
header, img_as_array = mx.recordio.unpack_img(item) # (210, 280,3)
img = Image.fromarray(img_as_array)
img.show()
normalized_img = (img_as_array - np.transpose(data_mean, (1, 2, 0)))/np.transpose(data_std, (1, 2, 0))
plt.matshow(normalized_img[:,:,0])
plt.show()
plt.matshow(normalized_img[:,:,1])
plt.show()
plt.matshow(normalized_img[:,:,2])
plt.show()
def read_from_recordio():
record = mx.recordio.MXRecordIO(RAW_PATH + "torcs_train.rec", "r")
record = mx.recordio.MXRecordIO(RAW_PATH + TRAIN_REC, "r")
for i in range(50):
item = record.read()
header, img = mx.recordio.unpack_img(item)
......@@ -60,8 +93,8 @@ def read_from_recordio():
def leveldb_to_rec(start_date):
train_record = mx.recordio.MXRecordIO(RAW_PATH + "torcs_train.rec", "w")
test_record = mx.recordio.MXRecordIO(RAW_PATH + "torcs_test.rec", "w")
train_record = mx.recordio.MXRecordIO(RAW_PATH + TRAIN_REC, "w")
test_record = mx.recordio.MXRecordIO(RAW_PATH + TEST_REC, "w")
keys = range(1, 484815)
train_keys, test_keys = train_test_split(keys,test_size=0.2)
......
GeneratedDpnetTrainingCode/CNNCreator_dpnet.py
View file @ a9103600
......@@ -84,7 +84,7 @@ class CNNCreator_dpnet:
def load_h5_files(self):
train_h5 = None
test_h5 = None
train_path = self._data_dir_ + "train_2.h5"
train_path = self._data_dir_ + "train.h5"
test_path = self._data_dir_ + "test.h5"
if os.path.isfile(train_path):
train_h5 = h5py.File(train_path, 'r')
......@@ -113,7 +113,7 @@ class CNNCreator_dpnet:
optimizer_params=(('learning_rate', 0.001),),
load_checkpoint=True,
context='gpu',
checkpoint_period=1,
checkpoint_period=5,
normalize=True):
if context == 'gpu':
mx_context = mx.gpu()
......@@ -161,8 +161,8 @@ class CNNCreator_dpnet:
self.module.fit(
train_data=train_iter,
eval_metric='mse',
eval_data=test_iter,
eval_metric='mse',
optimizer=optimizer,
optimizer_params=optimizer_params,
batch_end_callback=mx.callback.Speedometer(batch_size),
......@@ -334,8 +334,3 @@ class CNNCreator_dpnet:
data_names=self._input_names_,
label_names=self._output_names_,
context=context)
# print("start viz")
# graph = mx.viz.plot_network(predictions, shape={'image':(0,3,210,280), 'predictions_label':(0,14)}, node_attrs={"shape":'rect',"fixedsize":'false'})
# # graph.format = 'png'
# graph.render('graph')
GeneratedDpnetTrainingCode/CNNTrainer_Dpnet.py
View file @ a9103600
......@@ -5,18 +5,23 @@ import CNNCreator_dpnet
if __name__ == "__main__":
logging.basicConfig(level=logging.DEBUG)
logger = logging.getLogger()
handler = logging.FileHandler("train.log","w+", encoding=None, delay="true")
handler = logging.FileHandler("train.log","w", encoding=None, delay="true")
logger.addHandler(handler)
dpnet = CNNCreator_dpnet.CNNCreator_dpnet()
dpnet.train(
batch_size = 64,
num_epoch = 10,
num_epoch = 100,
context = 'cpu',
normalize = False,
normalize = True,
optimizer = 'sgd',
optimizer_params = {
'weight_decay': 5.0E-4,
'learning_rate': 0.01}
'learning_rate_minimum': 0.01,
'learning_rate_decay': 0.9,
'step_size': 8000}
)
GeneratedDpnetTrainingCode/custom_functions.py
View file @ a9103600
import mxnet as mx
import numpy as np
TRAIN_MEAN = [99.39394537, 110.60877108, 117.86127587]
TRAIN_STD = [42.04910545, 49.47874084, 62.61726178]
def load_data_rec(self, batch_size):
width = 280
height = 210
train_iter = mx.image.ImageIter(
path_imgrec=self._data_dir_ + "torcs_train.rec",
data_shape=(3, 210, 280), # (channels, height, width)
data_shape=(3, height, width), # (channels, height, width)
batch_size=batch_size,
label_width=14,
data_name='image',
......@@ -13,14 +19,14 @@ def load_data_rec(self, batch_size):
)
test_iter = mx.image.ImageIter(
path_imgrec=self._data_dir_ + "torcs_test.rec",
data_shape=(3, 210, 280), # (channels, height, width)
data_shape=(3, height, width), # (channels, height, width)
batch_size=batch_size,
label_width=14,
data_name='image',
label_name='predictions_label'
)
data_mean = None
data_std = None
data_mean = np.asarray([[[a] * width] * height for a in TRAIN_MEAN])
data_std = np.asarray([[[a] * width] * height for a in TRAIN_STD])
return train_iter, test_iter, data_mean, data_std
GeneratedDpnetTrainingCode/requirements-cpu.txt 0 → 100644
View file @ a9103600
mxnet
h5py
\ No newline at end of file
GeneratedDpnetTrainingCode/requirements_gpu.txt 0 → 100644
View file @ a9103600
mxnet-cu75
h5py
\ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment