added new custom loss, and plot visualization

analysis/full_visual_cmp.py

@@ -22,7 +22,7 @@ def main(idx):
     p_reg: Pipeline = None #Storable.load("evaluation/lohan/evaluation/final/output/REG_{}".format(idx))
     p_circle: Pipeline = None #Storable.load("evaluation/lohan/full_cmp/output/CIRCLE")
     p_quantile: Pipeline = None #Storable.load("evaluation/lohan/full_cmp/output/QUANTILE")
-    p_box: Pipeline = Storable.load("/Users/mariuslaska/Dropbox/BBOX_{}".format(idx)) # Storable.load("evaluation/lohan/evaluation/final/output/BBOX_{}".format(idx)) #Storable.load("../evaluation/UJIndoorLoc/grid/output/BBOX")##"evaluation/lohan/scaled_output_2/output/BBOX") # Storable.load("evaluation/gia/full_cmp/output/BBOX")
+    p_box: Pipeline = Storable.load("/home/laskama/Dropbox/BBOX_{}".format(idx)) # Storable.load("evaluation/lohan/evaluation/final/output/BBOX_{}".format(idx)) #Storable.load("../evaluation/UJIndoorLoc/grid/output/BBOX")##"evaluation/lohan/scaled_output_2/output/BBOX") # Storable.load("evaluation/gia/full_cmp/output/BBOX")
 
     fp_dims = (p_box.data_provider.floorplan_width, p_box.data_provider.floorplan_height)
 
@@ -132,10 +132,10 @@ def main(idx):
 
 
 def visualize(p_reg: Pipeline=None, p_circle: Pipeline=None, p_quantile: Pipeline=None, p_box: Pipeline=None, radius=9.0):
-    img = "../evaluation/gia/gia_floor_4.jpg"
-    fp_dims = (83.32, 17.16)
-    #img = resource_filename('data', 'lohan/CrowdsourcedDS1floor.png')
-    #fp_dims = (200, 80)
+    #img = "../evaluation/gia/gia_floor_4.jpg"
+    #fp_dims = (83.32, 17.16)
+    img = resource_filename('data', 'lohan/CrowdsourcedDS1floor.png')
+    fp_dims = (200, 80)
     #fp = FloorPlanPlotRec((83.32, 17.16), 20, floorplan_bg_img=img)
     # plot_data_heatmap(pipe, floor_plotter=fp)
     if p_reg is not None:
@@ -169,29 +169,29 @@ def visualize(p_reg: Pipeline=None, p_circle: Pipeline=None, p_quantile: Pipelin
     #     fp.show_plot()
 
     # fp = FloorPlanPlotRec(fp_dims, 2, floorplan_bg_img=img)
-    # for idx in range(50):#len(y_true)):
-    #     #idx += 150
-    #     #if np.prod(y_pred_box[idx, 2:]) < 600:
-    #     #    continue
-    #     fp = FloorPlanPlotRec(fp_dims, 2, floorplan_bg_img=img)
-    #     fp.draw_points(y_true[idx, 0], y_true[idx, 1], color='g', alpha=0.5)
-    #
-    #     # box_size = y_pred[:]
-    #     if p_circle is not None:
-    #         fp.draw_circles(centers=y_pred_circle[idx, :2], radius=y_pred_circle[idx, 2], color='r')
-    #     if p_reg is not None:
-    #         fp.draw_circles(centers=y_pred_reg[idx, :2], radius=radius, color='b')
-    #     if p_quantile is not None:
-    #         fp.draw_points(y_pred_quantile[idx, 0], y_pred_quantile[idx, 1], color="r")
-    #         fp.draw_points(y_pred_quantile[idx, 2], y_pred_quantile[idx, 3],
-    #                        color="b")
-    #
-    #         fp.draw_rectangles(anchors=y_pred_quantile[idx, :], color="g")
-    #     if p_box is not None:
-    #         fp.draw_rectangles_new(anchors=y_pred_box[idx, :], color='black')
-    #         #fp.draw_ellipse(y_pred_box[idx, :2], y_pred_box[idx, 2], y_pred_box[idx, 3], color="r")
-    #
-    #     fp.show_plot()
+    for idx in range(50):#len(y_true)):
+        idx += 150
+        #if np.prod(y_pred_box[idx, 2:]) < 600:
+        #    continue
+        fp = FloorPlanPlotRec(fp_dims, 2, floorplan_bg_img=img)
+        fp.draw_points(y_true[idx, 0], y_true[idx, 1], color='g', alpha=0.5)
+
+        # box_size = y_pred[:]
+        if p_circle is not None:
+            fp.draw_circles(centers=y_pred_circle[idx, :2], radius=y_pred_circle[idx, 2], color='r')
+        if p_reg is not None:
+            fp.draw_circles(centers=y_pred_reg[idx, :2], radius=radius, color='b')
+        if p_quantile is not None:
+            fp.draw_points(y_pred_quantile[idx, 0], y_pred_quantile[idx, 1], color="r")
+            fp.draw_points(y_pred_quantile[idx, 2], y_pred_quantile[idx, 3],
+                           color="b")
+
+            fp.draw_rectangles(anchors=y_pred_quantile[idx, :], color="g")
+        if p_box is not None:
+            fp.draw_rectangles_new(anchors=y_pred_box[idx, :], color='black')
+            #fp.draw_ellipse(y_pred_box[idx, :2], y_pred_box[idx, 2], y_pred_box[idx, 3], color="r")
+
+        fp.show_plot()
 
 
 if __name__ == "__main__":

analysis/plot_direction_of_error.py

+from il_pipeline.pipeline import Pipeline
+from il_pipeline.utility.storable import Storable
+
+import numpy as np
+from pkg_resources import resource_filename
+
+from analysis.evaluate_circle import PI
+from analysis.visualize_box_overlap import normalize
+import matplotlib.pyplot as plt
+
+from analysis.visualize_learning_progress import convert_from_grid, convert_from_2dim_grid
+from base.floor_plan_plot import FloorPlanPlotRec
+
+
+def main():
+    p_box: Pipeline = Storable.load("/home/laskama/Dropbox/BBOX_1")
+
+    dp = p_box.data_provider
+    fp_width = dp.floorplan_width
+    fp_height = dp.floorplan_height
+
+    split_idx = 1
+    radius = 10.069
+    sig_diff_threshold = 1.5
+    pos_diff_threshold = 100
+
+    x_test, y_true = dp.get_test_data(split_idx=split_idx, area_labels=False)
+    x_train, y_train = dp.get_train_data(split_idx=split_idx, area_labels=False)
+
+    # convert_from_grid(p_box)
+    convert_from_2dim_grid(p_box, grid_size=40)
+
+    y_pred = p_box.summary.y_pred[
+        split_idx]  # np.concatenate(p_box.summary.y_pred[split_idx], axis=0)
+    #y_pred_reg = p_reg.summary.y_pred[split_idx]
+
+    img = resource_filename('data', 'lohan/CrowdsourcedDS1floor.png')
+    # fp_dims = (83.32, 17.16)
+    fp_dims = (200, 80)
+
+    for idx in range(len(y_pred)):
+
+        sig_diff = np.linalg.norm(x_test - x_test[idx, :], axis=1)
+        pos_diff = np.linalg.norm(y_true - y_true[idx, :], axis=1)
+        sig_mask = sig_diff < sig_diff_threshold
+        pos_mask = pos_diff < pos_diff_threshold
+
+        mask = np.logical_and(sig_mask, pos_mask)
+        mask = np.where(mask)[0]
+
+        sub_pred = y_pred[mask, :]
+        sub_true = y_true[mask, :]
+
+        fp = FloorPlanPlotRec(fp_dims, 20, floorplan_bg_img=img, title="Error")
+
+        fp.draw_rectangles_new(anchors=y_pred[idx, :], color='black')
+        fp.draw_points(sub_pred[:, 0], sub_pred[:, 1], color="red")
+        fp.draw_points(sub_true[:, 0], sub_true[:, 1], color="blue")
+
+        fp.show_plot()
+
+if __name__ == "__main__":
+    main()
\ No newline at end of file

analysis/similarity_visualization.py

@@ -8,14 +8,14 @@ from analysis.evaluate_circle import PI
 from analysis.visualize_box_overlap import normalize
 import matplotlib.pyplot as plt
 
-from analysis.visualize_learning_progress import convert_from_grid
+from analysis.visualize_learning_progress import convert_from_grid, convert_from_2dim_grid
 from base.floor_plan_plot import FloorPlanPlotRec
 
 
 def main():
 
-    p_box: Pipeline = Storable.load("../evaluation/lohan/evaluation/deep/output/BBOX_NEW")
-    p_reg: Pipeline = Storable.load("../evaluation/lohan/evaluation/deep/output/REG")
+    p_box: Pipeline = Storable.load("/home/laskama/Dropbox/BBOX_1")
+    p_reg: Pipeline = Storable.load("/home/laskama/Dropbox/REG_1")
     dp = p_box.data_provider
     fp_width = dp.floorplan_width
     fp_height = dp.floorplan_height
@@ -26,7 +26,8 @@ def main():
     x_test, y_true = dp.get_test_data(split_idx=split_idx, area_labels=False)
     x_train, y_train = dp.get_train_data(split_idx=split_idx, area_labels=False)
 
-    convert_from_grid(p_box)
+    #convert_from_grid(p_box)
+    convert_from_2dim_grid(p_box, grid_size=40)
 
     y_pred = p_box.summary.y_pred[split_idx]#np.concatenate(p_box.summary.y_pred[split_idx], axis=0)
     y_pred_reg = p_reg.summary.y_pred[split_idx]
@@ -40,7 +41,7 @@ def main():
     size = np.prod(y_pred[:, 2:], axis=1)
     mask = np.where(size * 0.7 > radius**2 * PI)[0]
 
-    for idx in mask:
+    for idx in range(len(size)):
         # idx += 50
         draw(fp_width, fp_height, g_size, x_test, y_true, x_train, y_train, y_pred, y_pred_reg, idx)
 
@@ -83,10 +84,10 @@ def draw(fp_width, fp_height, g_size, x_test, y_true, x_train, y_train, y_pred_b
     my_cmap[:, -1] = 0.8
     my_cmap[0] = (.75, .75, .75, 0.8)
 
-    img = "../evaluation/gia/gia_floor_4.jpg"
-    #img = resource_filename('data', 'lohan/CrowdsourcedDS1floor.png')
-    fp_dims = (83.32, 17.16)
-    #fp_dims = (200, 80)
+    #img = "../evaluation/gia/gia_floor_4.jpg"
+    img = resource_filename('data', 'lohan/CrowdsourcedDS1floor.png')
+    #fp_dims = (83.32, 17.16)
+    fp_dims = (200, 80)
 
     fp = FloorPlanPlotRec(fp_dims, 20, floorplan_bg_img=img, title="Error")
 

analysis/visualize_box_overlap.py

@@ -18,12 +18,13 @@ def plot_mean_error_per_grid(pipe: Pipeline):
 
     p_c = pipe
 
-    convert_from_grid(pipe)
+    #convert_from_grid(pipe)
+    convert_from_2dim_grid(pipe, grid_size=40)
 
     dp = pipe.data_provider
     fp_height = dp.floorplan_height
     fp_width = dp.floorplan_width
-    g_size = 4
+    g_size = 16
 
     y_pred = np.concatenate(pipe.summary.y_pred, axis=0)
     y_true = np.concatenate(pipe.summary.y_true_labels, axis=0)
@@ -31,7 +32,7 @@ def plot_mean_error_per_grid(pipe: Pipeline):
     y_pred_reg = np.concatenate(p_c.summary.y_pred, axis=0)
 
     error = np.linalg.norm(y_pred_reg[:, :2] - y_true, axis=1)
-    mask = np.where(error <= 60)[0] #np.logical_and(error <= 10, y_true[:, 0] >= 40))[0]
+    mask = np.where(error <= 20)[0] #np.logical_and(error <= 10, y_true[:, 0] >= 40))[0]
     y_true = y_true[mask]
     y_pred = y_pred[mask]
     y_pred_reg = y_pred_reg[mask]
@@ -89,10 +90,10 @@ def plot_mean_error_per_grid(pipe: Pipeline):
     my_cmap[:, -1] = 0.8
     my_cmap[0] = (.75, .75, .75, 0.8)
 
-    img = "../evaluation/gia/gia_floor_4.jpg"
-    #img = resource_filename('data', 'lohan/CrowdsourcedDS1floor.png')
-    fp_dims = (83.32, 17.16)
-    #fp_dims = (200, 80)
+    #img = "../evaluation/gia/gia_floor_4.jpg"
+    img = resource_filename('data', 'lohan/CrowdsourcedDS1floor.png')
+    #fp_dims = (83.32, 17.16)
+    fp_dims = (200, 80)
 
     fp = FloorPlanPlotRec(fp_dims, 20, floorplan_bg_img=img, title="Error")
     fp_size = FloorPlanPlotRec(fp_dims, 20, floorplan_bg_img=img, title="BoxSize")
@@ -367,7 +368,6 @@ if __name__ == "__main__":
     #main()
     for id in [128]:
 
-        pipe: Pipeline = Storable.load(
-            "../evaluation/gia/evaluation/grid/output/BBOX_2")#{}".format(id))
+        pipe: Pipeline = Storable.load("/home/laskama/Dropbox/BBOX_1")
 
         plot_mean_error_per_grid(pipe)
\ No newline at end of file

base/custom_loss.py

@@ -2,6 +2,38 @@ import numpy as np
 import tensorflow as tf
 
 
+def hier_loss(y_true, y_pred):
+
+    g = 2
+
+    p_range = tf.range(tf.shape(y_pred)[1])
+
+    t_x = tf.cast(tf.mod(y_true[:, 0], g), dtype=tf.float32)
+    t_y = tf.cast(tf.cast(y_true[:, 0] / g, dtype=tf.int32), tf.float32)
+
+    t_x = tf.transpose(tf.ones([tf.shape(y_pred)[1], 1]) * t_x)
+    t_y = tf.transpose(tf.ones([tf.shape(y_pred)[1], 1]) * t_y)
+
+    p_x = tf.cast(tf.mod(p_range, g), dtype=tf.float32)
+    p_y = tf.cast(tf.cast(p_range / g, dtype=tf.int32), tf.float32)
+
+    p_x = tf.ones([tf.shape(y_true)[0], 1]) * p_x
+    p_y = tf.ones([tf.shape(y_true)[0], 1]) * p_y
+
+    t_x_offset = tf.transpose(tf.ones([tf.shape(y_pred)[1], 1]) * y_true[:, 1])
+    t_y_offset = tf.transpose(tf.ones([tf.shape(y_pred)[1], 1]) * y_true[:, 2])
+
+    x_diff = t_x - p_x + t_x_offset
+    y_diff = t_y - p_y + t_y_offset
+    
+    diff = tf.sqrt(tf.square(x_diff) + tf.square(y_diff))
+
+    loss = tf.multiply(diff, y_pred)
+
+    sum_loss = tf.reduce_sum(loss, axis=1)
+
+    return sum_loss
+
 def define_quantile_loss(params):
 
     def quantile_loss(y_true, y_pred):
@@ -767,6 +799,24 @@ def test_quantile_loss():
 
         writer.close()
 
+def test_hier_loss():
+    init = tf.global_variables_initializer()
+
+    y_true = np.array([[0, 0.2, 0.1], [0, 0.1, -0.3], [1, 0.1, 0.0]])
+
+    y_pred = np.array([[0.8, 0.1, 0.1, 0.0], [0.7, 0.2, 0.1, 0.0], [0.5, 0.3, 0.1, 0.1]])
+
+    y_true = tf.constant(y_true, dtype=tf.float32)
+    y_pred = tf.constant(y_pred, dtype=tf.float32)
+
+    with tf.Session() as sess:
+        # Run the initializer on `w`.
+        sess.run(init)
+        print(sess.run(hier_loss(y_true, y_pred)))
+        #writer = tf.summary.FileWriter("output", sess.graph)
+
+        #writer.close()
+
 
 def test_yolo_loss():
     init = tf.global_variables_initializer()
@@ -1064,21 +1114,23 @@ def test():
 
 if __name__ == "__main__":
 
-    test = np.array([[1,2,5,4,5,6], [1,2,5,4,5,6], [7,2,5,4,5,6], [1,2,5,4,7,6]])
-    y = np.array([[2, 1], [3, 0], [1, 1], [1, 1]])
-
-    test = test.reshape(len(test), 2, 3)
-
-    #print(test)
-
-    idx = y[:, 1]
+    test_hier_loss()
 
-    sub = test[:, idx, :][[0,1,2,3], [0,1,2,3]]
-    #print(sub)
-    #test()
-    test_yolo_loss()
-    #y_true = np.array([[7, 3], [2,4], [1, 1], [3,4]])
-
-    #y_pred = np.array([[6, 1, 7.5, 4], [1,3, 3, 4.5], [2, 0, 3, 1], [4,1,3,2]])
-
-    #custom_loss_python(y_pred, y_true)
\ No newline at end of file
+    # test = np.array([[1,2,5,4,5,6], [1,2,5,4,5,6], [7,2,5,4,5,6], [1,2,5,4,7,6]])
+    # y = np.array([[2, 1], [3, 0], [1, 1], [1, 1]])
+    #
+    # test = test.reshape(len(test), 2, 3)
+    #
+    # #print(test)
+    #
+    # idx = y[:, 1]
+    #
+    # sub = test[:, idx, :][[0,1,2,3], [0,1,2,3]]
+    # #print(sub)
+    # #test()
+    # test_yolo_loss()
+    # #y_true = np.array([[7, 3], [2,4], [1, 1], [3,4]])
+    #
+    # #y_pred = np.array([[6, 1, 7.5, 4], [1,3, 3, 4.5], [2, 0, 3, 1], [4,1,3,2]])
+    #
+    # #custom_loss_python(y_pred, y_true)
\ No newline at end of file

base/data_provider_base_grid.py

@@ -11,24 +11,45 @@ class DataProviderGridBase(DataProviderBase):
 
         self.grid_labels = None
 
-    def transform_to_2dim_grid_encoding(self, grid_size=10):
+    def transform_to_hier_grid_encoding(self, start_grid_size=10, division=2, depth=2):
+        labels = self.transform_to_2dim_grid_encoding(grid_size=start_grid_size)
 
-        self.grid_labels = np.zeros((len(self.labels), 3))
+        hier_labels = np.zeros((len(self.labels), 3, depth))
+
+        for d in range(depth):
+            if d > 0:
+                labels = self.transform_to_2dim_grid_encoding(
+                    grid_size=1/division, labels=labels)
+
+            hier_labels[:, :, d] = labels
+
+        self.grid_labels = hier_labels
+
+    def transform_to_2dim_grid_encoding(self, grid_size=10, labels=None):
+
+        if labels is None:
+            labels = self.labels
+
+        grid_labels = np.zeros((len(labels), 3))
 
         n_rows = math.ceil(self.floorplan_height/grid_size)
         n_cols = math.ceil(self.floorplan_width/grid_size)
 
-        r_id = (self.labels[:, 1] / grid_size).astype(int)
-        c_id = (self.labels[:, 0] / grid_size).astype(int)
+        r_id = (labels[:, 1] / grid_size).astype(int)
+        c_id = (labels[:, 0] / grid_size).astype(int)
 
         g_idx = r_id * n_cols + c_id
 
         # fix outliers
         g_idx[np.where(g_idx > (n_rows * n_cols - 1))] = (n_rows * n_cols - 1)
 
-        self.grid_labels[:, 2] = g_idx
+        grid_labels[:, 2] = g_idx
+
+        grid_labels[:, :2] = (labels % grid_size) / grid_size
+
+        self.grid_labels = grid_labels
 
-        self.grid_labels[:, :2] = (self.labels % grid_size) / grid_size
+        return grid_labels
 
     def transform_to_grid_encoding(self):
         self.grid_labels = np.zeros((len(self.labels), 3))

base/hier_model.py

+from il_pipeline.models.layer_generator import hidden_layers
+from il_pipeline.models.lr_normalizer import lr_normalizer
+from tensorflow import keras
+from tensorflow.keras import regularizers
+from tensorflow.keras.layers import Dense, Dropout
+from tensorflow.keras.models import Sequential
+
+
+def bbox_model_for_generator(metrics):
+
+    def define_classification_model_for_generator(X_cols, Y_cols, params) -> Sequential:
+
+        ap_input = keras.Input(shape=X_cols, name="AP")
+
+        first = keras.layers.Dense(params['first_neuron'], input_dim=X_cols,
+                        activation=params['activation'],
+                        kernel_regularizer=regularizers.l2(
+                            params['regularization_penalty'])
+                        )(ap_input)
+
+        first_output = keras.layers.Dense(Y_cols, activation=params['last_activation'],
+                        kernel_regularizer=regularizers.l2(
+                            params['regularization_penalty']))(first)
+
+        sec_input = keras.layers.concatenate([ap_input, first_output])
+
+        second = keras.layers.Dense(params['first_neuron'], input_dim=X_cols,
+                        activation=params['activation'],
+                        kernel_regularizer=regularizers.l2(
+                            params['regularization_penalty'])
+                        )(sec_input)
+
+        sec_output = keras.layers.Dense(Y_cols,
+                                          activation=params['last_activation'],
+                                          kernel_regularizer=regularizers.l2(
+                                              params['regularization_penalty']))(second)
+
+        model = Sequential()
+
+        model.add(Dense(params['first_neuron'], input_dim=X_cols,
+                        activation=params['activation'],
+                        kernel_regularizer=regularizers.l2(
+                            params['regularization_penalty'])
+                        ))
+
+        #model.add(BatchNormalization())
+
+        model.add(Dropout(params['dropout']))
+
+        hidden_layers(model, params, X_cols)
+
+        model.add(Dense(Y_cols, activation=params['last_activation'],
+                        #kernel_initializer=params['kernel_initializer'],
+                        kernel_regularizer=regularizers.l2(
+                            params['regularization_penalty'])))#,
+                        #kernel_initializer=initializers.RandomUniform(minval=-10, maxval=15)))#,
+                        #bias_initializer=initializers.RandomUniform(minval=0, maxval=0)))
+
+        # compile the model
+        model.compile(loss=params['losses'],
+                      metrics=[*metrics],
+                      optimizer=params['optimizer'](
+                          lr=lr_normalizer(params['lr'], params['optimizer'])))
+
+        return model
+
+
+    return define_classification_model_for_generator
\ No newline at end of file