informed loss plus started with encoding of walls

analysis/generate_walls.py

+from il_pipeline.pipeline import Pipeline
+from il_pipeline.pipeline import Storable
+from pkg_resources import resource_filename
+import numpy as np
+
+from base.data_provider_base_grid import DataProviderGridBase
+from base.floor_plan_plot import FloorPlanPlotRec
+
+
+def main():
+
+    img = resource_filename('data', 'lohan/CrowdsourcedDS1floor.png')
+    fp_dims = (200, 80)
+
+    fp = FloorPlanPlotRec(fp_dims, 5, floorplan_bg_img=img, walls_file="ver_walls.npy", add_walls=True)
+
+    fp.show_plot()
+    # plot_data_heatmap(pipe, floor_plotter=fp)
+
+
+def encode():
+    walls_h = np.load("hor_walls.npy")
+    walls_v = np.load("ver_walls.npy")
+
+    dp = DataProviderGridBase()
+    dp.encode_walls_to_2dim_overlapping_grid_encoding(walls_h, horizontal=True,
+                                                      grid_size=40, width=200,
+                                                      height=80)
+    dp.encode_walls_to_2dim_overlapping_grid_encoding(walls_v, horizontal=False,
+                                                      grid_size=40, width=200,
+                                                      height=80)
+    print("test")
+
+if __name__ == '__main__':
+    encode()
+    #main()

base/custom_loss.py

@@ -80,7 +80,6 @@ def define_circle_loss_try(params):
     def circle_loss(y_true, y_pred):
         loss = tf.zeros_like(y_true[:, 0])
 
-
         c_loss = tf.linalg.norm(tf.subtract(y_true, y_pred[:, :2]),
                                 axis=1)
         loss += c_loss
@@ -970,7 +969,7 @@ def define_yolo_loss_tanh_no_size(params):
     return yolo_loss
 
 
-def define_rotated_box_loss(params):
+def define_rotated_box_loss(params, walls_h, walls_v):
     import math as m
     tf_PI = tf.constant(m.pi)
 
@@ -1026,6 +1025,56 @@ def define_rotated_box_loss(params):
 
         y_pred_s = tf.add(y_pred_s, scale_add)
 
+        #
+        # Wall loss (before rotation)
+        #
+
+        # 1) compute distance from centers to walls (dist_walls)
+        walls = tf.concat([walls_h[:, :4], walls_v], axis=0)
+
+        nom = tf.matmul(
+            tf.reshape(y_pred_s[:, 0], [-1, 1]),
+            tf.reshape((walls[:, 3] - walls[:, 1]), [1, -1])) \
+              - tf.matmul(
+                  tf.reshape(y_pred_s[:, 1], [-1, 1]),
+                  tf.reshape((walls[:, 2] - walls[:, 0]), [1, -1])) \
+              + walls[:, 2] * walls[:, 1] \
+              - walls[:, 3] * walls[:, 0]
+
+        denom = tf.square(walls[:, 3] - walls[:, 1]) + tf.square(walls[:, 2] - walls[:, 0])
+        dist_walls = tf.abs(nom) / tf.sqrt(denom)
+
+        # 2) compute dist between horizontal box boundary and horizontal walls
+        d_hor = tf.transpose(tf.transpose(
+            dist_walls[:, :tf.shape(walls_h)[0]]) - y_pred_s[:, 3]/2)
+        d_ver = tf.transpose(tf.transpose(
+            dist_walls[:, tf.shape(walls_h)[0]:]) - y_pred_s[:, 2]/2)
+
+        wall_loss_hor = tf.reduce_min(tf.abs(d_hor), axis=1)
+        wall_loss_ver = tf.reduce_min(tf.abs(d_ver), axis=1)
+
+        loss += wall_loss_hor + wall_loss_ver
+
+        #
+        # Angle diff to horizontal walls (inversely weighted by distance)
+        #
+
+        angle_diff_scale = tf.transpose(
+            tf.transpose(
+                1/dist_walls[:, :tf.shape(walls_h)[0]]) *
+            tf.reduce_min(
+                dist_walls[:, :tf.shape(walls_h)[0]], axis=1))
+
+        angle_diff = angle_diff_scale * (
+                tf.reshape(y_pred_s[:, 4], [-1, 1]) -
+                tf.reshape(walls_h[:, 4], [1, -1]))
+
+        loss += tf.reduce_sum(angle_diff, axis=1)
+
+        #
+        # following operations are computed with rotated local coordinate system
+        #
+
         # rotate (c_x, c_y) and (t_x, t_y) by omega * PI around center of grid cell
         omega = y_pred_s[:, 4]
 
@@ -1049,22 +1098,6 @@ def define_rotated_box_loss(params):
 
         box_loss += c_loss
 
-        # if "size" in params:
-        #     size_loss = tf.reduce_sum(tf.square(y_pred_s[:, 2:]), axis=1)
-        #
-        #     if "threshold" in params["size"]:
-        #         # only count size loss if greater than 50.0
-        #         threshold = params["size"]["threshold"]
-        #         size_loss = tf.maximum(tf.zeros_like(size_loss),
-        #                                tf.subtract(size_loss, tf.multiply(
-        #                                    tf.ones_like(size_loss), threshold)))
-        #
-        #     if "scale" in params["size"]:
-        #         factor = params["size"]["scale"]
-        #         size_loss = tf.multiply(size_loss, tf.constant(factor))
-        #
-        #     box_loss += size_loss
-
         if "outside" in params:
 
             delta = 20.0
@@ -1081,11 +1114,10 @@ def define_rotated_box_loss(params):
                 tf.divide(y_pred_s[:, 3], tf.constant(delta))),
                 tf.constant(0.0))
 
-            outside_x_loss = outside_x # tf.maximum(tf.zeros_like(outside_x), outside_x)
-            outside_y_loss = outside_y # tf.maximum(tf.zeros_like(outside_y), outside_y)
+            outside_x_loss = outside_x
+            outside_y_loss = outside_y
 
             outside_loss = tf.square(outside_x_loss) + tf.square(outside_y_loss)
-            #outside_loss = outside_x_loss + outside_y_loss
 
             if "scale" in params["outside"]:
                 factor = params["outside"]["scale"]
@@ -1336,6 +1368,9 @@ def test_yolo_loss():
 
     y_pred = np.array([[0.7, 0.3, 0.5, 0.2, 0, 0.6, 0.4, 0.2, 0.9, 0.1,0, -0.4], [0.3, 0.2, 0.4, 0.1, 0, -0.2, 0.2, 0.2, 0.6, 0.1,0, -0.8]])
 
+    walls_h = np.array([[-0.7, 0.2, 0.5, 0.2, 0], [0.6, 0.4, 0.2, 0.9, 0.1], [-0.7, 0.2, 0.5, 0.1, -0.5]])
+    walls_v = np.array([[-0.7, 0.2, 0.5, 0.1], [-0.6, 0.4, 0.2, 0.9]])
+
     #c_sub = y_pred[:, [0::, 1]*5]
 
     #test = y_pred[[0,1,0,0], :]
@@ -1343,8 +1378,14 @@ def test_yolo_loss():
 
     y_true = tf.constant(y_true, dtype=tf.float32)
     y_pred = tf.constant(y_pred, dtype=tf.float32)
-
-    loss_func = define_rotated_box_loss(params=dict(grid={"scale": 4.0}, size={"scale": 0.0055}, outside={"scale": 0.1}))
+    walls_h_tf = tf.constant(walls_h, dtype=tf.float32)
+    walls_v_tf = tf.constant(walls_v, dtype=tf.float32)
+
+    loss_func = define_rotated_box_loss(
+        params=dict(grid={"scale": 4.0},
+                    size={"scale": 0.0055},
+                    outside={"scale": 0.1}),
+        walls_h=walls_h_tf, walls_v=walls_v_tf)
 
     with tf.Session() as sess:
         # Run the initializer on `w`.

base/data_provider_base_grid.py

@@ -15,6 +15,8 @@ class DataProviderGridBase(DataProviderBase):
 
         self.grid_labels = None
         self.grid_size = None
+        self.grid_walls_h = None
+        self.grid_walls_v = None
 
     def transform_to_hier_grid_encoding(self, start_grid_size=10, division=2, depth=3):
         coord_center = (0.5, 0.5)
@@ -199,6 +201,100 @@ class DataProviderGridBase(DataProviderBase):
         self.grid_labels = grid_encoding
         self.aug_encoding = aug_encoding
 
+    def encode_walls_to_2dim_overlapping_grid_encoding(self, walls, horizontal=True, overlap_strategy="ignore", grid_size=10.0, padding_ratio=0.1, height=None, width=None):
+
+        img = resource_filename('data', 'lohan/CrowdsourcedDS1floor.png')
+        fp = FloorPlanPlotRec((200, 80), 2, floorplan_bg_img=img)
+
+        if height is None:
+            height = self.floorplan_height
+
+        if width is None:
+            width = self.floorplan_width
+
+        # Determine #row,col for 1st layer
+        num_row_l1 = np.ceil(height / grid_size) + 1
+        num_col_l1 = np.ceil(width / grid_size) + 1
+        num_l1 = int(num_row_l1 * num_col_l1)
+
+        num = num_l1
+
+        origins = []
+
+        # Determine centers & origins (lower-left)
+        for idx in range(num):
+
+            # 1st layer
+            r_idx = int(idx / num_col_l1)
+            c_idx = idx % num_col_l1
+
+            x = c_idx * grid_size
+            y = r_idx * grid_size
+
+            origins.append(np.array([x, y]))
+            fp.axis.text(x+1, y-2, str(idx))
+
+        # Determine closest origin for labels (the corresponding cell is
+        # responsible for encoding
+        o_plt = np.array([[o[0], o[1], 40, 40] for o in origins])
+        o_plt_pad = np.array([[o[0], o[1], 48, 48] for o in origins])
+
+        origins = np.array(origins)
+
+        fp.draw_rectangles_new(o_plt, color="black")
+        fp.draw_rectangles_new(o_plt_pad, color="grey")
+
+        fp.draw_points(origins[:, 0], origins[:, 1], color="black")
+
+        dist_line_center = np.full((len(walls), len(origins)), np.inf)
+        dist_to_center = np.full((len(walls), len(origins), 2), np.inf)
+        dist_to_center_cmp = np.full((len(walls), len(origins), 2, 2), np.inf)
+        for o_idx, o in enumerate(origins):
+            d_1 = np.linalg.norm(walls[:, :2] - o, axis=1)
+            d_2 = np.linalg.norm(walls[:, 2:4] - o, axis=1)
+            d_ceter_line = np.linalg.norm(np.mean(np.stack(
+                [walls[:, [0, 2]],
+                 walls[:, [1, 3]]], axis=2), axis=1) - o, axis=1)
+
+            dist_to_center[:, o_idx, 0] = d_1
+            dist_to_center[:, o_idx, 1] = d_2
+            dist_line_center[:, o_idx] = d_ceter_line
+
+            dist_to_center_cmp[:, o_idx, 0, :] = np.abs(walls[:, :2] - o)
+            dist_to_center_cmp[:, o_idx, 1, :] = np.abs(walls[:, 2:4] - o)
+
+        resp_grid_center_line_idx = np.argmin(dist_line_center, axis=1)
+        resp_grid_center_origins = origins[resp_grid_center_line_idx, :]
+
+        # resp_grid_idx = np.argmin(dist_to_center, axis=1)
+        # resp_origins = origins[resp_grid_idx, :]
+
+        # compute dist to respective center
+        p1_dist = walls[:, :2] - resp_grid_center_origins
+        p1_inside = np.all(p1_dist < grid_size + grid_size * padding_ratio, axis=1)
+        p2_dist = walls[:, 2:4] - resp_grid_center_origins
+        p2_inside = np.all(p2_dist < grid_size + grid_size * padding_ratio, axis=1)
+
+        # idx of walls that cannot be encoded within single grid_cell
+        mask = np.logical_and(p1_inside, p2_inside)
+        split_required = np.where(~mask)[0]
+        no_split_required = np.where(mask)[0]
+        if overlap_strategy == "ignore":
+            walls = walls[no_split_required, :]
+            resp_grid_center_line_idx = resp_grid_center_line_idx[no_split_required]
+            resp_grid_center_origins = resp_grid_center_origins[no_split_required, :]
+
+        # determine final encoding
+        grid_encoding = np.zeros((len(walls[:, :2]), 5))
+        grid_encoding[:, 4] = resp_grid_center_line_idx
+        grid_encoding[:, :2] = (walls[:, :2] - resp_grid_center_origins) / (grid_size / 2.0 + grid_size * padding_ratio)
+        grid_encoding[:, 2:4] = (walls[:, 2:4] - resp_grid_center_origins) / (grid_size / 2.0 + grid_size * padding_ratio)
+
+        if horizontal:
+            self.grid_walls_h = grid_encoding
+        else:
+            self.grid_walls_v = grid_encoding
+
     def transform_to_2dim_3layer_grid_encoding(self, grid_size=10.0, height=None, width=None, labels=None):
 
         if labels is None: