Commit 185a79a9 authored by Svetlana Pavlitskaya's avatar Svetlana Pavlitskaya

Dpnet traning. Data preparation scripts. Script to test Dpnet.

parent 0a2262df
......@@ -2,7 +2,7 @@ configuration Dpnet{
num_epoch : 100
batch_size : 64
context:cpu
normalize: true
normalize: false
optimizer : sgd{
learning_rate : 0.01
weight_decay : 0.0005
......
......@@ -4,3 +4,5 @@ venv/
.idea/
.git
*.iml
*.pyc
*.log
import mxnet as mx
def load_data_rec(self, batch_size):
train_iter = mx.image.ImageIter(
path_imgrec=self._data_dir_ + "torcs_train.rec",
data_shape=(3, 210, 280), # (channels, height, width)
batch_size=batch_size,
label_width=14,
data_name='image',
label_name='predictions_label'
)
test_iter = mx.image.ImageIter(
path_imgrec=self._data_dir_ + "torcs_test.rec",
data_shape=(3, 210, 280), # (channels, height, width)
batch_size=batch_size,
label_width=14,
data_name='image',
label_name='predictions_label'
)
data_mean = None
data_std = None
return train_iter, test_iter, data_mean, data_std
import matplotlib.pyplot as plt
import numpy as np
import re
NETWORK= "Dpnet"
DATASET = "DeepDriving Dataset"
# define the paths to the training logs
logs = [
(0, "train_1.log") # lr=1e-2
# (85, "training_65.log"), # lr=1e-3
# (100, "training_85.log"), # lr=1e-4
]
# initialize the list of train rank-1 and rank-5 accuracies, along
# with the training loss
(trainRank1, trainRank5, trainLoss) = ([], [], [])
# initialize the list of validation rank-1 and rank-5 accuracies,
# along with the validation loss
(valRank1, valRank5, valLoss) = ([], [], [])
# loop over the training logs
for (i, (endEpoch, p)) in enumerate(logs):
# load the contents of the log file, then initialize the batch
# lists for the training and validation data
rows = open(p).read().strip()
(bTrainMSE, bTrainRank5, bTrainLoss) = ([], [], [])
(bValRank1, bValRank5, bValLoss) = ([], [], [])
# grab the set of training epochs
epochs = set(re.findall(r'Epoch\[(\d+)\]', rows))
epochs = sorted([int(e) for e in epochs])
# loop over the epochs
for e in epochs:
# find all rank-1 accuracies, rank-5 accuracies, and loss
# values, then take the final entry in the list for each
s = r'Epoch\[' + str(e) + '\].*Train-mse=(.*)'
mse = re.findall(s, rows)[-1]
# update the batch training lists
bTrainMSE.append(float(mse))
# extract the validation rank-1 and rank-5 accuracies for each
# epoch, followed by the loss
bValRank1 = re.findall(r'Validation-mse=(.*)', rows)
# convert the validation rank-1, rank-5, and loss lists to floats
bValRank1 = [float(x) for x in bValRank1]
# check to see if we are examining a log file other than the
# first one, and if so, use the number of the final epoch in
# the log file as our slice index
if i > 0 and endEpoch is not None:
trainEnd = endEpoch - logs[i - 1][0]
valEnd = endEpoch - logs[i - 1][0]
# otherwise, this is the first epoch so no subtraction needs
# to be done
else:
trainEnd = endEpoch
valEnd = endEpoch
# update the training lists
trainRank1.extend(bTrainMSE[0:trainEnd])
trainRank5.extend(bTrainRank5[0:trainEnd])
trainLoss.extend(bTrainLoss[0:trainEnd])
# update the validation lists
valRank1.extend(bValRank1[0:valEnd])
valRank5.extend(bValRank5[0:valEnd])
valLoss.extend(bValLoss[0:valEnd])
# plot the accuracies
plt.style.use("ggplot")
plt.figure()
plt.plot(np.arange(0, len(trainRank1)), trainRank1,
label="train_rank1")
plt.plot(np.arange(0, len(trainRank5)), trainRank5,
label="train_rank5")
plt.plot(np.arange(0, len(valRank1)), valRank1,
label="val_rank1")
plt.plot(np.arange(0, len(valRank5)), valRank5,
label="val_rank5")
plt.title("{}: rank-1 and rank-5 accuracy on {}".format(
NETWORK, DATASET))
plt.xlabel("Epoch #")
plt.ylabel("Accuracy")
plt.legend(loc="lower right")
# plot the losses
plt.style.use("ggplot")
plt.figure()
plt.plot(np.arange(0, len(trainLoss)), trainLoss,
label="train_loss")
plt.plot(np.arange(0, len(valLoss)), valLoss,
label="val_loss")
plt.title("{}: cross-entropy loss on {}".format(NETWORK, DATASET))
plt.xlabel("Epoch #")
plt.ylabel("Loss")
plt.legend(loc="upper right")
plt.show()
\ No newline at end of file
......@@ -5,4 +5,6 @@ protobuf
leveldb
tables
glog
mxnet
\ No newline at end of file
mxnet
opencv-python
sklearn
\ No newline at end of file
import csv
import cv2
import numpy as np
import matplotlib.pyplot as plt
from matplotlib.ticker import FormatStrFormatter
import mxnet as mx
import os
EXAMPLES_PATH = "/media/sveta/4991e634-dd81-4cb9-bf46-2fa9c7159263/TORCS_examples/"
MODEL_PATH = "model/dpnet1/dpnet_newest"
RAW_PATH = "/media/sveta/4991e634-dd81-4cb9-bf46-2fa9c7159263/TORCS_raw/"
def main():
# Load saved checkpoint
sym, arg_params, aux_params = mx.model.load_checkpoint(MODEL_PATH, 0)
mod = mx.mod.Module(symbol=sym,
context=mx.cpu(),
data_names=['image'],
label_names=['predictions_label'])
mod.bind(for_training=False,
data_shapes=[('image', (1, 3, 210, 280))],
label_shapes=mod._label_shapes)
mod.set_params(arg_params, aux_params, allow_missing=True)
# Get Test Data
files = [f for f in os.listdir(EXAMPLES_PATH) if f.endswith(".png")]
for f in files:
key = f[:-4]
img = get_image(key)
labels = get_labels(key)
# Predict
from collections import namedtuple
Batch = namedtuple('Batch', ['data'])
mod.forward(Batch([mx.nd.array(img)]))
prob = mod.get_outputs()[0].asnumpy()
prob = prob[0].tolist()
# Plot ground truth against predicted
plt.plot(range(len(labels)), labels, label='Ground truth')
plt.plot(range(len(prob)), prob, label='Predicted')
plt.legend()
ax = plt.gca()
ax.yaxis.set_major_formatter(FormatStrFormatter('%.2f'))
plt.show()
def get_image(key):
filename = EXAMPLES_PATH + key + ".png"
img = cv2.imread(filename) # read image in b,g,r order
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) # change to r,g,b order
# img = cv2.resize(img, (224, 224)) # resize to 224*224 to fit model
img = np.swapaxes(img, 0, 2)
img = np.swapaxes(img, 1, 2) # change to (channel, height, width)
img = img[np.newaxis, :] # extend to (example, channel, heigth, width)
return img
def get_labels(key):
labels_file = EXAMPLES_PATH + key + "_labels.csv"
with open(labels_file, 'rb') as csvfile:
csv_reader = csv.reader(csvfile, delimiter=' ', quotechar='|')
labels = [row for row in csv_reader][0]
labels = [float(a) for a in labels]
return labels
main()
import caffe
from caffe.proto import caffe_pb2
import csv
import cv2
import datetime
import h5py
import mxnet as mx
import numpy as np
from PIL import Image
import plyvel
from sklearn.cross_validation import train_test_split
import tarfile
import os
ARCHIVE = True
ARCHIVE = False
CHUNK_SIZE = 10000
LEVELDB_PATH = "/media/sveta/4991e634-dd81-4cb9-bf46-2fa9c7159263/TORCS_Training_1F"
HDF5_PATH = "/media/sveta/4991e634-dd81-4cb9-bf46-2fa9c7159263/TORCS_HDF5_2"
HDF5_PATH = "/media/sveta/4991e634-dd81-4cb9-bf46-2fa9c7159263/TORCS_HDF5_3/"
RAW_PATH = "/media/sveta/4991e634-dd81-4cb9-bf46-2fa9c7159263/TORCS_raw/"
EXAMPLES_PATH = "/media/sveta/4991e634-dd81-4cb9-bf46-2fa9c7159263/TORCS_examples/"
def main():
start_date = datetime.datetime.now()
db = plyvel.DB(LEVELDB_PATH, paranoid_checks=True, create_if_missing=False)
datum = caffe_pb2.Datum()
all_images = []
all_indicators = []
file_idx = 1
# leveldb_to_rec(start_date)
read_from_recordio()
# compute_train_mean()
def compute_train_mean():
record = mx.recordio.MXRecordIO(RAW_PATH + "torcs_train.rec", "r")
img_stats = []
for i in range(50):
item = record.read()
header, img_as_array = mx.recordio.unpack_img(item)
# img is RGB of shape (210, 280, 3)
# img = Image.fromarray(img_as_array)
# img.show()
mean, std = cv2.meanStdDev(img_as_array)
mean_as_img = Image.fromarray(mean)
mean_as_img.show()
img_stats.append(np.array([mean[::-1] / 255, std[::-1] / 255]))
img_stats = np.mean(img_stats, axis=0)
print(img_stats)
def read_from_recordio():
record = mx.recordio.MXRecordIO(RAW_PATH + "torcs_train.rec", "r")
for i in range(50):
item = record.read()
header, img = mx.recordio.unpack_img(item)
key = str(header[2])
convert_to_image_and_save(img, key)
labels_file = EXAMPLES_PATH + key + "_labels.csv"
with open(labels_file, 'wb') as csvfile:
spamwriter = csv.writer(csvfile, delimiter=' ',
quotechar='|', quoting=csv.QUOTE_MINIMAL)
spamwriter.writerow(header[1].tolist())
def leveldb_to_rec(start_date):
train_record = mx.recordio.MXRecordIO(RAW_PATH + "torcs_train.rec", "w")
test_record = mx.recordio.MXRecordIO(RAW_PATH + "torcs_test.rec", "w")
keys = range(1, 484815)
train_keys, test_keys = train_test_split(keys,test_size=0.2)
print str(len(train_keys)) + " samples for training"
print str(len(test_keys)) + " samples for testing"
db, datum = read_db()
for key, value in db:
key_as_int = int(float(key))
datum = datum.FromString(value)
indicators = np.array(datum.float_data, dtype='f')
indicators = normalize(indicators)
image = caffe.io.datum_to_array(datum).astype(np.float32) # shape is (3, 210, 280)
all_images.append(image)
all_indicators.append(indicators)
if len(all_images) >= CHUNK_SIZE:
print("File " + str(file_idx))
write_to_hdf5(all_images, all_indicators, file_idx, start_date)
all_images = []
all_indicators = []
file_idx += 1
# final file
print("File " + str(file_idx))
write_to_hdf5(all_images, all_indicators, file_idx, start_date)
image_data = caffe.io.datum_to_array(datum) # shape is (3, 210, 280)
image_data = np.transpose(image_data, (1, 2, 0)) # shape is (210, 280, 3)
image_data = image_data[:, :, ::-1] # BGR to RGB
# convert_to_image_and_save(image_data, key_as_str)
header = mx.recordio.IRHeader(0, indicators, key_as_int, 0)
image_record = mx.recordio.pack_img(header, image_data, img_fmt='.png')
if key_as_int in train_keys:
train_record.write(image_record)
elif key_as_int in test_keys:
test_record.write(image_record)
else:
raise Exception("Unknown key " + key)
if key_as_int % 1000 == 0:
print str(key_as_int) + "/" + str(len(keys))
current_time = datetime.datetime.now()
elapsed_time = current_time - start_date
print("\t Total time spent: " + str(elapsed_time))
def read_db():
db = plyvel.DB(LEVELDB_PATH, paranoid_checks=True, create_if_missing=False)
datum = caffe_pb2.Datum()
return db, datum
def convert_to_image_and_save(image_data, key):
img = Image.fromarray(image_data)
img.save(EXAMPLES_PATH + key + ".png")
def write_to_hdf5(images, indicators, file_idx, start_date):
......@@ -91,6 +163,34 @@ def normalize_value(old_value, old_min, old_max):
return new_value
def leveldb_to_hdf5(start_date):
db, datum = read_db()
all_images = []
all_indicators = []
file_idx = 1
for key, value in db:
datum = datum.FromString(value)
indicators = np.array(datum.float_data, dtype='f')
indicators = normalize(indicators)
image_data = caffe.io.datum_to_array(datum) # .astype(np.float32) # shape is (3, 210, 280)
image_data = np.transpose(image_data, (1, 2, 0))
image_data = image_data[:, :, ::-1]
all_images.append(image_data)
all_indicators.append(indicators)
if len(all_images) >= CHUNK_SIZE:
print("File " + str(file_idx))
write_to_hdf5(all_images, all_indicators, file_idx, start_date)
all_images = []
all_indicators = []
file_idx += 1
# final file
print("File " + str(file_idx))
write_to_hdf5(all_images, all_indicators, file_idx, start_date)
main()
import datetime
import h5py
import os
HDF5_PATH = "/media/sveta/4991e634-dd81-4cb9-bf46-2fa9c7159263/TORCS_HDF5/"
start_date = datetime.datetime.now()
big_file = h5py.File(HDF5_PATH + "all_train.h5")
dset_image = big_file.create_dataset("image", (0, 3, 210, 280), maxshape=(None, 3,210,280), chunks=True)
dset_labels = big_file.create_dataset("predictions_label", (0, 13), maxshape=(None, 13), chunks=True)
data_start_idx = 0
for file_num in range(1, 485):
file_path = HDF5_PATH + "train_" + str(file_num) + ".h5"
print(file_path)
with h5py.File(file_path, "r") as f:
images = f["image"]
labels = f["predictions_label"]
dset_image.resize( (dset_image.shape[0]+1000, 3,210,280) )
dset_image[data_start_idx:data_start_idx+1000,:,:,:] = images
dset_labels.resize( (dset_labels.shape[0]+1000, 13) )
dset_labels[data_start_idx:data_start_idx+1000,:] = labels
data_start_idx += 1000
os.remove(file_path)
end_time = datetime.datetime.now()
elapsed_time = end_time - start_date
print("Elapsed time " + str(elapsed_time))
model/
__pycache__/
venv/
.idea/
.git
*.iml
*.pyc
*.log
......@@ -7,6 +7,8 @@ import h5py
import sys
import numpy as np
import custom_functions
@mx.init.register
class MyConstant(mx.init.Initializer):
def __init__(self, value):
......@@ -18,7 +20,7 @@ class MyConstant(mx.init.Initializer):
class CNNCreator_dpnet:
module = None
_data_dir_ = "/media/sveta/4991e634-dd81-4cb9-bf46-2fa9c7159263/TORCS_HDF5/"
_data_dir_ = "/media/sveta/4991e634-dd81-4cb9-bf46-2fa9c7159263/TORCS_raw/"
_model_dir_ = "model/dpnet/"
_model_prefix_ = "dpnet"
_input_names_ = ['image']
......@@ -82,7 +84,7 @@ class CNNCreator_dpnet:
def load_h5_files(self):
train_h5 = None
test_h5 = None
train_path = self._data_dir_ + "train_1.h5"
train_path = self._data_dir_ + "train_2.h5"
test_path = self._data_dir_ + "test.h5"
if os.path.isfile(train_path):
train_h5 = h5py.File(train_path, 'r')
......@@ -111,7 +113,7 @@ class CNNCreator_dpnet:
optimizer_params=(('learning_rate', 0.001),),
load_checkpoint=True,
context='gpu',
checkpoint_period=5,
checkpoint_period=1,
normalize=True):
if context == 'gpu':
mx_context = mx.gpu()
......@@ -136,7 +138,8 @@ class CNNCreator_dpnet:
del optimizer_params['learning_rate_decay']
train_iter, test_iter, data_mean, data_std = self.load_data(batch_size)
# train_iter, test_iter, data_mean, data_std = self.load_data(batch_size)
train_iter, test_iter, data_mean, data_std = custom_functions.load_data_rec(self, batch_size)
if self.module == None:
if normalize:
self.construct(mx_context, data_mean, data_std)
......@@ -331,8 +334,8 @@ class CNNCreator_dpnet:
data_names=self._input_names_,
label_names=self._output_names_,
context=context)
print("start viz")
graph = mx.viz.plot_network(predictions, shape={'image':(0,3,210,280), 'predictions_label':(0,14)}, node_attrs={"shape":'rect',"fixedsize":'false'})
# graph.format = 'png'
graph.render('graph')
# print("start viz")
# graph = mx.viz.plot_network(predictions, shape={'image':(0,3,210,280), 'predictions_label':(0,14)}, node_attrs={"shape":'rect',"fixedsize":'false'})
# # graph.format = 'png'
# graph.render('graph')
......@@ -5,21 +5,18 @@ import CNNCreator_dpnet
if __name__ == "__main__":
logging.basicConfig(level=logging.DEBUG)
logger = logging.getLogger()
handler = logging.FileHandler("train.log","w", encoding=None, delay="true")
handler = logging.FileHandler("train.log","w+", encoding=None, delay="true")
logger.addHandler(handler)
dpnet = CNNCreator_dpnet.CNNCreator_dpnet()
dpnet.train(
batch_size = 64,
num_epoch = 100,
num_epoch = 10,
context = 'cpu',
normalize = True,
normalize = False,
optimizer = 'sgd',
optimizer_params = {
'weight_decay': 5.0E-4,
'learning_rate': 0.01}
)
import mxnet as mx
def load_data_rec(self, batch_size):
train_iter = mx.image.ImageIter(
path_imgrec=self._data_dir_ + "torcs_train.rec",
data_shape=(3, 210, 280), # (channels, height, width)
batch_size=batch_size,
label_width=14,
data_name='image',
label_name='predictions_label'
)
test_iter = mx.image.ImageIter(
path_imgrec=self._data_dir_ + "torcs_test.rec",
data_shape=(3, 210, 280), # (channels, height, width)
batch_size=batch_size,
label_width=14,
data_name='image',
label_name='predictions_label'
)
data_mean = None
data_std = None
return train_iter, test_iter, data_mean, data_std
target/
output/
venv/
.idea/
.git
*.iml
\ No newline at end of file
*.iml
src/main/resources/dpnet/
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="EntryPointsManager">
<entry_points version="2.0" />
</component>
<component name="MavenProjectsManager">
<option name="originalFiles">
<list>
<option value="$PROJECT_DIR$/pom.xml" />
</list>
</option>
</component>
<component name="ProjectRootManager" version="2" languageLevel="JDK_1_8" project-jdk-name="1.8" project-jdk-type="JavaSDK">
<output url="file://$PROJECT_DIR$/classes" />
</component>
</project>
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment