Data preparations scripts and updated EMADL model

DataPreparation/binaryproto_to_numpy.py

+import caffe
+import csv
+import numpy as np
+from PIL import Image
+import sys
+
+blob = caffe.proto.caffe_pb2.BlobProto()
+data = open("./driving_mean_1F.binaryproto" , 'rb' ).read()
+blob.ParseFromString(data)
+data = np.array(blob.data)
+arr = np.array( caffe.io.blobproto_to_array(blob) )
+arr = arr[0]
+# np.save("./mean_image.npy", arr[0]) # shape is (210, 280, 3)
+
+np.savetxt("./mean_image_R.txt", arr[0])
+np.savetxt("./mean_image_G.txt", arr[1])
+np.savetxt("./mean_image_B.txt", arr[2])
\ No newline at end of file

DataPreparation/custom_functions.py

-import mxnet as mx
-
-
-def load_data_rec(self, batch_size):
-
-    train_iter = mx.image.ImageIter(
-        path_imgrec=self._data_dir_ + "torcs_train.rec",
-        data_shape=(3, 210, 280),  # (channels, height, width)
-        batch_size=batch_size,
-        label_width=14,
-        data_name='image',
-        label_name='predictions_label'
-    )
-    test_iter = mx.image.ImageIter(
-        path_imgrec=self._data_dir_ + "torcs_test.rec",
-        data_shape=(3, 210, 280),  # (channels, height, width)
-        batch_size=batch_size,
-        label_width=14,
-        data_name='image',
-        label_name='predictions_label'
-    )
-    data_mean = None
-    data_std = None
-
-
-    return train_iter, test_iter, data_mean, data_std

DataPreparation/log_parser.py

@@ -2,101 +2,62 @@ import matplotlib.pyplot as plt
 import numpy as np
 import re
 
-NETWORK= "Dpnet"
-DATASET = "DeepDriving Dataset"
-
-# define the paths to the training logs
-logs = [
-    (0, "train_1.log")			# lr=1e-2
-    # (85, "training_65.log"),		# lr=1e-3
-    # (100, "training_85.log"),		# lr=1e-4
-]
-
-# initialize the list of train rank-1 and rank-5 accuracies, along
-# with the training loss
-(trainRank1, trainRank5, trainLoss) = ([], [], [])
-
-# initialize the list of validation rank-1 and rank-5 accuracies,
-# along with the validation loss
-(valRank1, valRank5, valLoss) = ([], [], [])
-
-# loop over the training logs
-for (i, (endEpoch, p)) in enumerate(logs):
-    # load the contents of the log file, then initialize the batch
-    # lists for the training and validation data
-    rows = open(p).read().strip()
-    (bTrainMSE, bTrainRank5, bTrainLoss) = ([], [], [])
-    (bValRank1, bValRank5, bValLoss) = ([], [], [])
-
-    # grab the set of training epochs
-    epochs = set(re.findall(r'Epoch\[(\d+)\]', rows))
-    epochs = sorted([int(e) for e in epochs])
-    # loop over the epochs
-    for e in epochs:
-        # find all rank-1 accuracies, rank-5 accuracies, and loss
-        # values, then take the final entry in the list for each
-        s = r'Epoch\[' + str(e) + '\].*Train-mse=(.*)'
-        mse = re.findall(s, rows)[-1]
-
-        # update the batch training lists
-        bTrainMSE.append(float(mse))
-
-        # extract the validation rank-1 and rank-5 accuracies for each
-        # epoch, followed by the loss
-        bValRank1 = re.findall(r'Validation-mse=(.*)', rows)
+EPOCH_WISE = False
+NETWORK= "DPNet"
+LOGFILE = "../../dpnet_weights/normalized/train.log"
+STEP = 50
+
+rows = open(LOGFILE).read().strip()
+train_mse = list()
+validation_mse = list()
+train_iterations = list()
+validation_iterations = list()
+speeds = list()
+
+# grab the set of training epochs
+epochs = set(re.findall(r'Epoch\[(\d+)\]', rows))
+epochs = sorted([int(e) for e in epochs])
+for e in epochs:
+    train_mse_regexp = r'Epoch\[' + str(e) + '\].*(\s)mse=(.*)'
+    mse = re.findall(train_mse_regexp, rows)
+    mse = [float(a[1]) for a in mse]
+    if EPOCH_WISE:
+        train_mse.append(mse[-1])
+    else:
+        train_mse += mse
 
-        # convert the validation rank-1, rank-5, and loss lists to floats
-        bValRank1 = [float(x) for x in bValRank1]
+    speed_regexp = r'Epoch\[' + str(e) + '\].*(\s)Speed: (.*) samples'
+    speed = re.findall(speed_regexp, rows)
+    speed = [float(a[1]) for a in speed]
+    speeds += speed
 
-        # check to see if we are examining a log file other than the
-        # first one, and if so, use the number of the final epoch in
-        # the log file as our slice index
-        if i > 0 and endEpoch is not None:
-            trainEnd = endEpoch - logs[i - 1][0]
-            valEnd = endEpoch - logs[i - 1][0]
+    validation_mse_regexp = r'Epoch\[' + str(e) + '\].*Validation-mse=(.*)'
+    current_validation_mse = re.findall(validation_mse_regexp, rows)
+    validation_mse.append(float(current_validation_mse[0]))
 
-        # otherwise, this is the first epoch so no subtraction needs
-        # to be done
+    last_iteration = train_iterations[-1] if len(train_iterations) > 0 else 0
+    if EPOCH_WISE:
+        train_iterations.append(e)
+        validation_iterations.append(e)
     else:
-            trainEnd = endEpoch
-            valEnd = endEpoch
-
-        # update the training lists
-        trainRank1.extend(bTrainMSE[0:trainEnd])
-        trainRank5.extend(bTrainRank5[0:trainEnd])
-        trainLoss.extend(bTrainLoss[0:trainEnd])
+        current_iterations = range(last_iteration+STEP, last_iteration+STEP * len(mse) + STEP, STEP)
+        train_iterations += current_iterations
+        validation_iterations.append(last_iteration+STEP * len(mse) + STEP)
 
-        # update the validation lists
-        valRank1.extend(bValRank1[0:valEnd])
-        valRank5.extend(bValRank5[0:valEnd])
-        valLoss.extend(bValLoss[0:valEnd])
+print("Mean speed is " + str(np.mean(speeds)))
 
 # plot the accuracies
-plt.style.use("ggplot")
-plt.figure()
-plt.plot(np.arange(0, len(trainRank1)), trainRank1,
-         label="train_rank1")
-plt.plot(np.arange(0, len(trainRank5)), trainRank5,
-         label="train_rank5")
-plt.plot(np.arange(0, len(valRank1)), valRank1,
-         label="val_rank1")
-plt.plot(np.arange(0, len(valRank5)), valRank5,
-         label="val_rank5")
-plt.title("{}: rank-1 and rank-5 accuracy on {}".format(
-    NETWORK, DATASET))
-plt.xlabel("Epoch #")
-plt.ylabel("Accuracy")
-plt.legend(loc="lower right")
-
-# plot the losses
-plt.style.use("ggplot")
+# plt.style.use("ggplot")
 plt.figure()
-plt.plot(np.arange(0, len(trainLoss)), trainLoss,
-         label="train_loss")
-plt.plot(np.arange(0, len(valLoss)), valLoss,
-         label="val_loss")
-plt.title("{}: cross-entropy loss on {}".format(NETWORK, DATASET))
-plt.xlabel("Epoch #")
-plt.ylabel("Loss")
+plt.plot(train_iterations, train_mse,
+         label="train")
+plt.plot(validation_iterations, validation_mse,
+         label="validation")
+if EPOCH_WISE:
+    plt.xlabel("Epochs #")
+else:
+    plt.xlabel("Iterations")
+plt.ylabel("MSE")
 plt.legend(loc="upper right")
+plt.grid()
 plt.show()
\ No newline at end of file

DataPreparation/test_dpnet.py

@@ -7,7 +7,7 @@ import mxnet as mx
 import os
 
 EXAMPLES_PATH = "/media/sveta/4991e634-dd81-4cb9-bf46-2fa9c7159263/TORCS_examples/"
-MODEL_PATH = "model/dpnet1/dpnet_newest"
+MODEL_PATH = "../../dpnet_weights/normalized/dpnet_newest"
 RAW_PATH = "/media/sveta/4991e634-dd81-4cb9-bf46-2fa9c7159263/TORCS_raw/"
 
 
@@ -16,10 +16,10 @@ def main():
     sym, arg_params, aux_params = mx.model.load_checkpoint(MODEL_PATH, 0)
     mod = mx.mod.Module(symbol=sym,
                         context=mx.cpu(),
-                        data_names=['image'],
+                        data_names=['data'],
                         label_names=['predictions_label'])
     mod.bind(for_training=False,
-             data_shapes=[('image', (1, 3, 210, 280))],
+             data_shapes=[('data', (1, 3, 210, 280))],
              label_shapes=mod._label_shapes)
 
     mod.set_params(arg_params, aux_params, allow_missing=True)
@@ -39,9 +39,10 @@ def main():
         prob = prob[0].tolist()
 
         # Plot ground truth against predicted
-        plt.plot(range(len(labels)), labels, label='Ground truth')
-        plt.plot(range(len(prob)), prob, label='Predicted')
+        plt.scatter(range(len(labels)), labels, marker='x', label='Ground truth')
+        plt.scatter(range(len(prob)), prob, marker = 'x', label='Predicted')
         plt.legend()
+        plt.grid()
         ax = plt.gca()
         ax.yaxis.set_major_formatter(FormatStrFormatter('%.2f'))
         plt.show()

DataPreparation/torcs_data_preparation.py

@@ -4,6 +4,7 @@ import csv
 import cv2
 import datetime
 import h5py
+import math
 import matplotlib.pyplot as plt
 import mxnet as mx
 import numpy as np
@@ -13,6 +14,8 @@ from sklearn.cross_validation import train_test_split
 import tarfile
 import os
 
+TRAIN_LENGTH = 387851
+
 TEST_REC = "torcs_test.rec"
 TRAIN_REC = "torcs_train.rec"
 
@@ -32,14 +35,16 @@ def main():
     # leveldb_to_rec(start_date)
     # read_from_recordio()
     # compute_train_mean()
-    test_normalization()
+    # test_normalization()
+    # check_saved_labels()
+    check_saved_images()
 
 
 def compute_train_mean():
     record = mx.recordio.MXRecordIO(RAW_PATH + TRAIN_REC, "r")
     all_means = list()
     all_std = list()
-    for i in range(387851):
+    for i in range(TRAIN_LENGTH):
         if i % 1000 == 0:
             print(i)
         item = record.read()
@@ -91,6 +96,28 @@ def read_from_recordio():
                                     quotechar='|', quoting=csv.QUOTE_MINIMAL)
             spamwriter.writerow(header[1].tolist())
 
+def check_saved_labels():
+    record = mx.recordio.MXRecordIO(RAW_PATH + TRAIN_REC, "r")
+    for i in range(TRAIN_LENGTH):
+        item = record.read()
+        header, img = mx.recordio.unpack_img(item)
+        affordance = header[1].tolist()
+        # if not any([True if a>=0.1 and a<=0.9 else False for a in affordance ]):
+        if any(math.isnan(item) for item in affordance):
+            print(affordance)
+
+def check_saved_images():
+    record = mx.recordio.MXRecordIO(RAW_PATH + TRAIN_LENGTH, "r")
+    for i in range(TRAIN_LENGTH):
+        item = record.read()
+        header, img = mx.recordio.unpack_img(item)
+        try:
+            img = Image.fromarray(img)
+            img.verify()
+        except (IOError, SyntaxError) as e:
+            print('Bad file:', i)
+
+
 
 def leveldb_to_rec(start_date):
     train_record = mx.recordio.MXRecordIO(RAW_PATH + TRAIN_REC, "w")

EMADLModels/generate_for_safetynet.sh

+#!/usr/bin/env bash
+echo "Generating files.."
+java -jar embedded-montiarc-emadl-generator-0.2.1-SNAPSHOT-jar-with-dependencies.jar -m src/models -r Safetynet -o generated

EMADLModels/src/models/Dpnet.cnnt

+configuration Dpnet{
+     num_epoch : 100
+     batch_size : 64
+     context:cpu
+     normalize: true
+     optimizer : sgd{
+	     learning_rate: 0.01
+         // reduce the learning rate starting from 0.01 every 8000 iterations by a factor of 0.9 (decrease by 10%)
+	     learning_rate_decay: 0.9
+         step_size: 8000
+     }
+
+}

CNNModels/src/models/Dpnet.emadl → EMADLModels/src/models/Dpnet.emadl

 component Dpnet{
-    ports in Z(0:255)^{3, 210, 280} image,
+    ports in Z(0:255)^{3, 210, 280} data,
          out Q(-oo:oo)^{14,1,1} predictions;
 
     implementation CNN {
@@ -15,7 +15,7 @@ component Dpnet{
             Dropout()
         }
 
-        image ->
+        data ->
         conv(kernel=(11,11), channels=96, convStride=(4,4)) ->
         conv(kernel=(5,5), channels=256, convStride=(4,4)) ->
         conv(kernel=(3,3), channels=384, hasPool=false) ->
@@ -23,7 +23,7 @@ component Dpnet{
         conv(kernel=(3,3), channels=256) ->
         fc() -> 
         fc() -> 
-        FullyConnected(units=14, no_bias=true) ->
+        FullyConnected(units=2, no_bias=true) ->
         predictions
 
     }

CNNModels/src/models/Dpnet.cnnt → EMADLModels/src/models/Safetynet.cnnt

-configuration Dpnet{
+configuration Safetynet{
      num_epoch : 100
      batch_size : 64
      context:cpu
      normalize: true
      optimizer : sgd{
-         weight_decay : 0.0005
+	learning_rate: 0.01
          // reduce the learning rate starting from 0.01 every 8000 iterations by a factor of 0.9 (decrease by 10%)
 	 learning_rate_decay: 0.9
          step_size: 8000
-         learning_rate_minimum: 0.01
+         weight_decay : 0.0005
      }
 
 }

EMADLModels/src/models/Safetynet.emadl

+component Safetynet{
+    ports in Z(0:255)^{3, 210, 280} data,
+         out Q(0:1)^{2,1,1} predictions;
+
+    implementation CNN {
+
+        def conv(kernel, channels, hasPool=true, convStride=(1,1)){
+            Convolution(kernel=kernel, channels=channels, stride=convStride) ->
+            Relu() ->
+            Pooling(pool_type="max", kernel=(3,3), stride=(2,2), ?=hasPool)
+        }
+        def fc(){
+            FullyConnected(units=4096) ->
+            Relu() ->
+            Dropout()
+        }
+
+        image ->
+        conv(kernel=(11,11), channels=96, convStride=(4,4)) ->
+        conv(kernel=(5,5), channels=256, convStride=(4,4)) ->
+        conv(kernel=(3,3), channels=384, hasPool=false) ->
+        conv(kernel=(3,3), channels=384, hasPool=false) ->
+        conv(kernel=(3,3), channels=256) ->
+        fc() ->
+        fc() ->
+        FullyConnected(units=2) ->
+	Softmax() ->
+        predictions
+
+    }
+}

GeneratedDpnetTrainingCode/CNNCreator_dpnet.py

@@ -23,7 +23,7 @@ class CNNCreator_dpnet:
     _data_dir_ = "/media/sveta/4991e634-dd81-4cb9-bf46-2fa9c7159263/TORCS_raw/"
     _model_dir_ = "model/dpnet/"
     _model_prefix_ = "dpnet"
-    _input_names_ = ['image']
+    _input_names_ = ['data']
     _input_shapes_ = [(3,210,280)]
     _output_names_ = ['predictions_label']
 
@@ -139,7 +139,7 @@ class CNNCreator_dpnet:
 
 
         # train_iter, test_iter, data_mean, data_std = self.load_data(batch_size)
-        train_iter, test_iter, data_mean, data_std = custom_functions.load_data_rec(self, batch_size)
+        train_iter, test_iter, data_mean, data_std = custom_functions.load_data_rec(self._data_dir_, batch_size)
         if self.module == None:
             if normalize:
                 self.construct(mx_context, data_mean, data_std)
@@ -174,18 +174,18 @@ class CNNCreator_dpnet:
 
 
     def construct(self, context, data_mean=None, data_std=None):
-        image = mx.sym.var("image",
+        image = mx.sym.var("data",
                            shape=(0,3,210,280))
         # image, output shape: {[3,210,280]}
 
         if not data_mean is None:
-            assert(not data_std is None)
+            # assert(not data_std is None)
             _data_mean_ = mx.sym.Variable("_data_mean_", shape=(3,210,280), init=MyConstant(value=data_mean.tolist()))
             _data_mean_ = mx.sym.BlockGrad(_data_mean_)
-            _data_std_ = mx.sym.Variable("_data_std_", shape=(3,210,280), init=MyConstant(value=data_mean.tolist()))
-            _data_std_ = mx.sym.BlockGrad(_data_std_)
+            #_data_std_ = mx.sym.Variable("_data_std_", shape=(3,210,280), init=MyConstant(value=data_mean.tolist()))
+            #_data_std_ = mx.sym.BlockGrad(_data_std_)
             image = mx.symbol.broadcast_sub(image, _data_mean_)
-            image = mx.symbol.broadcast_div(image, _data_std_)
+            # image = mx.symbol.broadcast_div(image, _data_std_)
         conv1_ = mx.symbol.pad(data=image,
                                mode='constant',
                                pad_width=(0,0,0,0,5,4,4,3),
@@ -323,13 +323,25 @@ class CNNCreator_dpnet:
                                       p=0.5,
                                       name="dropout7_")
         fc7_ = mx.symbol.FullyConnected(data=dropout7_,
+                                        num_hidden=256,
+                                        no_bias=False,
+                                        name="fc7_")
+        relu8_ = mx.symbol.Activation(data=fc7_,
+                                      act_type='relu',
+                                      name="relu8_")
+
+        dropout8_ = mx.symbol.Dropout(data=relu8_,
+                                      p=0.5,
+                                      name="dropout8_")
+        fc8_ = mx.symbol.FullyConnected(data=dropout8_,
                                         num_hidden=14,
                                         no_bias=True,
-                                        name="fc7_")
+                                        name="fc8_")
 
-        predictions = mx.symbol.LinearRegressionOutput(data=fc7_,
+        predictions = mx.symbol.LinearRegressionOutput(data=fc8_,
                                                        name="predictions")
 
+
         self.module = mx.mod.Module(symbol=mx.symbol.Group([predictions]),
                                     data_names=self._input_names_,
                                     label_names=self._output_names_,

GeneratedDpnetTrainingCode/CNNTrainer_Dpnet.py

@@ -5,7 +5,7 @@ import CNNCreator_dpnet
 if __name__ == "__main__":
     logging.basicConfig(level=logging.DEBUG)
     logger = logging.getLogger()
-    handler = logging.FileHandler("train.log","w", encoding=None, delay="true")
+    handler = logging.FileHandler("train.log","a", encoding=None, delay="true")
     logger.addHandler(handler)
 
     dpnet = CNNCreator_dpnet.CNNCreator_dpnet()
@@ -16,12 +16,12 @@ if __name__ == "__main__":
         normalize = True,
         optimizer = 'sgd',
         optimizer_params = {
-            'weight_decay': 5.0E-4,
-            'learning_rate_minimum': 0.01,
+            'learning_rate': 0.01,
             'learning_rate_decay': 0.9,
             'step_size': 8000}
 
 
 
+
     )
 

GeneratedDpnetTrainingCode/cluster_job.sh

+#!/usr/bin/env zsh
+
+### Job name
+#BSUB -J TrainDPNET
+
+### File / path where STDOUT & STDERR will be written
+###    %J is the job ID, %I is the array ID
+#BSUB -o train_dpnet.%J.%I
+
+#BSUB -B
+#BSUB -N
+#BSUB -u svetlana.pavlitskaya@rwth-aachen.de
+
+### Request the time you need for execution in minutes
+### Format [hour:]minute,
+### that means for 80 minutes you could also use this: 1:20
+#BSUB -W 10:00
+
+### Request vitual memory (in MB)
+#BSUB -M 4096
+
+### Request GPU Queue
+#BSUB -gpu -
+#BSUB -R gpu
+
+
+module load cuda
+
+hostname
+nvcc --version
+
+export MXNET_CPU_WORKER_NTHREADS=32
+export MXNET_ENGINE_TYPE=ThreadedEnginePerDevice
+
+cd /home/sp705423/GeneratedDpnetTrainingCode
+pip install --user -r requirements.txt
+python CNNTrainer_Dpnet.py

GeneratedDpnetTrainingCode/custom_functions.py

 import mxnet as mx
 import numpy as np
 
+DATA_NAME = 'data'
+PREDICTIONS_LABEL = 'predictions_label'
+
 TRAIN_MEAN = [99.39394537, 110.60877108, 117.86127587]
 TRAIN_STD = [42.04910545, 49.47874084, 62.61726178]
 
-
-def load_data_rec(self, batch_size):
+def load_data_rec(data_dir, batch_size):
     width = 280
     height = 210
 
+    data_mean = np.load("./mean_image.npy")
     train_iter = mx.image.ImageIter(
-        path_imgrec=self._data_dir_ + "torcs_train.rec",
+        path_imgrec=data_dir + "torcs_train.rec",
         data_shape=(3, height, width),  # (channels, height, width)
         batch_size=batch_size,
         label_width=14,
-        data_name='image',
-        label_name='predictions_label'
+        data_name=DATA_NAME,
+        label_name=PREDICTIONS_LABEL
     )
     test_iter = mx.image.ImageIter(
-        path_imgrec=self._data_dir_ + "torcs_test.rec",
+        path_imgrec=data_dir + "torcs_test.rec",
         data_shape=(3, height, width),  # (channels, height, width)
         batch_size=batch_size,
         label_width=14,
-        data_name='image',
-        label_name='predictions_label'
+        data_name=DATA_NAME,
+        label_name=PREDICTIONS_LABEL
     )
 
-    data_mean = np.asarray([[[a] * width] * height for a in TRAIN_MEAN])
-    data_std = np.asarray([[[a] * width] * height for a in TRAIN_STD])
+    data_std = None
+    # data_mean = np.asarray([[[a] * width] * height for a in TRAIN_MEAN])
+    # data_std = np.asarray([[[a] * width] * height for a in TRAIN_STD])
 
     return train_iter, test_iter, data_mean, data_std

GeneratedDpnetTrainingCode/finetune_pretrained_alexnet.py

+import mxnet as mx
+
+import custom_functions
+
+CONTEXT = mx.cpu()
+MODEL_PATH = "model/alexnet_pretrained/caffenet"
+MODEL_PATH_FINETUNED = "model/alexnet_finetuned/caffenet"
+DATA_DIR = "/media/sveta/4991e634-dd81-4cb9-bf46-2fa9c7159263/TORCS_raw/"
+MODEL_PREFIX = "dpnet"
+
+batch_size = 64
+num_epoch = 1
+begin_epoch = 0
+
+symbol, arg_params, aux_params =