Commit fff9845c authored by Marius Laska's avatar Marius Laska

new loss function (without size, only balances center error to box size ratio

parent 1bc428ea
......@@ -2,31 +2,58 @@ import numpy as np
from il_pipeline.pipeline import Pipeline
from il_pipeline.utility.storable import Storable
import matplotlib.pyplot as plt
import pandas as pd
import seaborn as sns
from analysis.full_visual_cmp import move_pred_closer
from analysis.plot_analysis import get_avg_box_size
from analysis.visualize_learning_progress import \
convert_from_2dim_overlapping_grid
from main import calc_acc_c
import os
floor = 0
f_idx = None
def main():
def main(res_dict):
fig_title = ""# "Lohan (fold-{})".format(f_idx + 1)
plt.figure()
f2 = "../evaluation/UJIndoorLoc/b0/f2/final/output/REG_{}"
plt.title(fig_title)#, floor: {}".format(floor))
plt.xlabel("Area ["+r'$m^2$' + "]")
plt.ylabel("ACC")
f2 = "/home/laskama/Dropbox/evaluation/UJIndoorLoc/b0/f{}/final/output/REG_{}"
dist_list = []
size_list = []
mse_list = []
#f2 = "/home/laskama/Dropbox/evaluation/lohan/REG_{}"
for idx in range(10):
p: Pipeline = Storable.load(f2.format(idx+1))
p: Pipeline = Storable.load(f2.format(floor, idx+1))
# move_pred_closer(p, dist=2.8)
y_true = np.concatenate(p.summary.y_true_labels, axis=0)
y_pred = np.concatenate(p.summary.y_pred, axis=0)
if f_idx is None:
y_true = np.concatenate(p.summary.y_true_labels, axis=0)
y_pred = np.concatenate(p.summary.y_pred, axis=0)
else:
y_true = p.summary.y_true_labels[f_idx]
y_pred = p.summary.y_pred[f_idx]
n = len(y_true)
dist = np.sort(np.linalg.norm(y_pred - y_true, axis=1))
res_dict["Model"] += ["DNN (3L)"] * len(dist)
res_dict["MSE"] += dist.tolist()
res_dict["fold"] += [f_idx] * len(dist)
mse_list.append(np.mean(dist))
size = np.pi * np.square(dist)
acc = [i/n for i in range(n)]
......@@ -36,35 +63,161 @@ def main():
# plt.plot(size, acc)
size_avg = np.mean(np.stack(size_list, axis=1), axis=1)
dist_avg = np.mean(np.stack(dist_list, axis=1), axis=1)
#plt.plot(dist_avg, [i / n for i in range(n)])
#plt.xlabel("Error [m]")
#plt.ylabel("CDF")
plt.plot(size_avg, [i / n for i in range(n)], linestyle="dashed", color="green", label="DNN->O (3L)")
#plt.show()
# plt.figure()
# dist_avg = np.mean(np.stack(dist_list, axis=1), axis=1)
# plt.boxplot(dist_avg)
print("REG_MSE: {}".format(np.mean(np.array(mse_list))))
f2 = "/home/laskama/Dropbox/evaluation/UJIndoorLoc/b0/f{}/final/output/REG_1l_{}"
dist_list = []
size_list = []
mse_list = []
#f2 = "/home/laskama/Dropbox/evaluation/lohan/REG_1l_{}"
for idx in range(10):
p: Pipeline = Storable.load(f2.format(floor, idx + 1))
# move_pred_closer(p, dist=2.8)
if f_idx is None:
y_true = np.concatenate(p.summary.y_true_labels, axis=0)
y_pred = np.concatenate(p.summary.y_pred, axis=0)
else:
y_true = p.summary.y_true_labels[f_idx]
y_pred = p.summary.y_pred[f_idx]
n = len(y_true)
dist = np.sort(np.linalg.norm(y_pred - y_true, axis=1))
res_dict["Model"] += ["DNN (1L)"] * len(dist)
res_dict["MSE"] += dist.tolist()
res_dict["fold"] += [f_idx] * len(dist)
mse_list.append(np.mean(dist))
size = np.pi * np.square(dist)
acc = [i / n for i in range(n)]
dist_list.append(dist)
size_list.append(size)
# plt.plot(size, acc)
size_avg = np.mean(np.stack(size_list, axis=1), axis=1)
plt.plot(size_avg, [i / n for i in range(n)], linestyle="dashed",
color="red", label="DNN->O (1L)")
print("REG_MSE: {}".format(np.mean(np.array(mse_list))))
f2 = "/home/laskama/Dropbox/evaluation/UJIndoorLoc/b0/f{}/final/output/REG_2l_{}"
dist_list = []
size_list = []
mse_list = []
#f2 = "/home/laskama/Dropbox/evaluation/lohan/REG_2l_{}"
for idx in range(10):
p: Pipeline = Storable.load(f2.format(floor, idx + 1))
# move_pred_closer(p, dist=2.8)
if f_idx is None:
y_true = np.concatenate(p.summary.y_true_labels, axis=0)
y_pred = np.concatenate(p.summary.y_pred, axis=0)
else:
y_true = p.summary.y_true_labels[f_idx]
y_pred = p.summary.y_pred[f_idx]
n = len(y_true)
dist = np.sort(np.linalg.norm(y_pred - y_true, axis=1))
res_dict["Model"] += ["DNN (2L)"] * len(dist)
res_dict["MSE"] += dist.tolist()
res_dict["fold"] += [f_idx] * len(dist)
mse_list.append(np.mean(dist))
size = np.pi * np.square(dist)
acc = [i / n for i in range(n)]
dist_list.append(dist)
size_list.append(size)
# plt.plot(size, acc)
size_avg = np.mean(np.stack(size_list, axis=1), axis=1)
plt.plot(size_avg, [i / n for i in range(n)], linestyle="dashed",
color="blue", label="DNN->O (2L)")
print("REG_MSE: {}".format(np.mean(np.array(mse_list))))
plt.plot(size_avg, [i/n for i in range(n)], linestyle="dashed")
# compare BBOX values
f1 = "../evaluation/UJIndoorLoc/b0/f2/final/output/BBOX_{}l_{}o{}_{}"
for aug in ["_aug", ""]:
f1 = "/home/laskama/Dropbox/evaluation/lohan/BBOX_{}l_{}o{}_{}"
f1 = "/home/laskama/Dropbox/evaluation/UJIndoorLoc/b0/f{}/final/output/BBOX_{}l_{}o{}_{}"
f1 = "/home/laskama/PycharmProjects/bboxPrediction/evaluation/UJIndoorLoc/b0/f{}/final/output/BBOX_{}l_{}{}_{}"
mse_list = []
for aug in ["_aug"]:#["_aug", ""]:
for l in [1,2]:#, 2]:
for o in [5, 4, 3]:
for o in [5, 10, 15, 20, 25, 30]:#[5, 4, 3]:
b_size_list = []
acc_box_list = []
for idx in [1,2,3]:
file = f1.format(l, o, aug, idx)
for idx in [1,2,3,4,5]:
#file = f1.format(floor, l, o, aug, idx)
file = f1.format(floor, l, o, aug, idx)
if not os.path.exists(file):
print(file)
continue
# print(file)
p: Pipeline = Storable.load(file)
convert_from_2dim_overlapping_grid(p, grid_size=40, quantile=False,
store=True,
average_samples=True)
y_true = np.concatenate(p.summary.y_true_labels, axis=0)
y_pred = np.concatenate(p.summary.y_pred, axis=0)
if f_idx is None:
y_true = np.concatenate(p.summary.y_true_labels, axis=0)
y_pred = np.concatenate(p.summary.y_pred, axis=0)
else:
y_true = p.summary.y_true_labels[f_idx]
y_pred = p.summary.y_pred[f_idx]
acc_box, wrong_mask, correct_mask = calc_acc_c(y_true, y_pred)
b_size = get_avg_box_size(p)
acc_box_list.append(acc_box)
b_size_list.append(b_size)
dist = np.linalg.norm(y_pred[:, :2] - y_true, axis=1)
mse_list.append(dist)
res_dict["Model"] += ["BBOX ({}L{})".format(l, aug)] * len(dist)
res_dict["MSE"] += dist.tolist()
res_dict["fold"] += [f_idx] * len(dist)
plt_label = ""
if o == 5:
plt_label = "BBOX ({}L{})".format(l, aug)
if aug == "_aug":
c = "r"
......@@ -77,13 +230,37 @@ def main():
m = "^"
elif l == 3:
m = ">"
plt.scatter(b_size, acc_box, color=c, marker=m)
# plt.scatter(b_size, acc_box, color=c, marker=m)
# plot mean of 3 attemps
plt.scatter(np.mean(np.array(b_size_list)), np.mean(np.array(acc_box_list)), color=c, marker=m)
plt.show()
plt.scatter(np.mean(np.array(b_size_list)), np.mean(np.array(acc_box_list)), color=c, marker=m, label=plt_label)
print("BBOX_MSE: {}".format(np.mean(np.array(mse_list))))
#plt.ylim(0.4, 1.0)
plt.xlim(0, 600.0)
plt.legend(loc="lower right")
#plt.show()
return res_dict
#plt.savefig(fig_title + ".pdf")
if __name__ == "__main__":
main()
\ No newline at end of file
res_dict = {"Model": [], "MSE": [], "fold": []}
f_idx = None
for idx in range(1):
floor = idx
#f_idx = None
res_dict = main(res_dict)
df = pd.DataFrame(res_dict)
plt.figure()
sns.boxplot("Model", "MSE", data=df, color="skyblue")
plt.show()
\ No newline at end of file
from il_pipeline.pipeline import Pipeline
from il_pipeline.utility.storable import Storable
import numpy as np
from analysis.correlation_knn import evaluate_correlation
from analysis.visualize_learning_progress import \
convert_from_2dim_overlapping_grid
def main():
path_lohan = "/home/laskama/Dropbox/evaluation/lohan/BBOX_{}l_{}o{}_{}"
path_uji = "/home/laskama/Dropbox/evaluation/UJIndoorLoc/b0/f0/final/output/BBOX_{}l_{}o{}_{}"
for o in [3, 4, 5]:
for num_layers in range(2):
pred_pos = None
true_pos = None
size = None
for sample in range(5):
try:
f_path = path_lohan.format(num_layers+1, o, "_aug", sample + 1)
model: Pipeline = Storable.load(f_path)
except:
continue
convert_from_2dim_overlapping_grid(model, grid_size=40, quantile=False,
store=True,
average_samples=True)
pred = np.concatenate(model.summary.y_pred, axis=0)
true = np.concatenate(model.summary.y_true_labels, axis=0)
p_pos = pred[:, :2]
p_size = np.prod(pred[:, 2:], axis=1)
if pred_pos is None:
pred_pos = p_pos
true_pos = true
else:
pred_pos = np.concatenate((pred_pos, p_pos), axis=0)
true_pos = np.concatenate((true_pos, true), axis=0)
if size is None:
size = p_size
else:
size = np.concatenate((size, p_size), axis=0)
summary = {"y": [], "beta": [], "p_val": [], "r_2": [], "f": [], "type": []}
error = np.linalg.norm(pred_pos - true_pos, axis=1)
evaluate_correlation(size, error, "error -> size", summary)
print("num_layers: {}".format(num_layers +1))
print("o: {}".format(o))
print(summary)
print("\n")
if __name__ == "__main__":
main()
......@@ -2,8 +2,8 @@ from il_pipeline.pipeline import Pipeline
from il_pipeline.utility.storable import Storable
import numpy as np
import pandas as pd
from mgwr.gwr import GWR, MGWR, GWRResults
from mgwr.sel_bw import Sel_BW
#from mgwr.gwr import GWR, MGWR, GWRResults
#from mgwr.sel_bw import Sel_BW
from shapely.geometry import Point
from pkg_resources import resource_filename
......@@ -12,12 +12,15 @@ from statsmodels.regression.linear_model import OLS
from statsmodels.tools import add_constant
from sklearn.linear_model import LinearRegression
import matplotlib.pyplot as plt
from mpl_toolkits.axes_grid1 import make_axes_locatable
from sklearn import linear_model
from base.floor_plan_plot import FloorPlanPlotRec
def main(dir_p="../evaluation/lohan/evaluation/new_knn/output/", f_idx=0):
bbox: Pipeline = Storable.load(dir_p + "BBOX")
f = "/home/laskama/PycharmProjects/bboxPrediction/evaluation/UJIndoorLoc/b0/f0/final/output/BBOX_1l_20_aug_1"
bbox: Pipeline = Storable.load(f)#dir_p + "BBOX_1l_3o_aug_1")
convert_from_2dim_overlapping_grid(bbox, grid_size=40, quantile=False,
store=True,
......@@ -25,7 +28,8 @@ def main(dir_p="../evaluation/lohan/evaluation/new_knn/output/", f_idx=0):
#convert_from_2dim_grid(bbox, grid_size=40, quantile=False, store=True,
# average_samples=True)
reg: Pipeline = Storable.load(dir_p + "REG_DEEP") # Storable.load("../evaluation/simulation/evaluation/big_2/output/" + "REG") #Storable.load("../evaluation/simulation/evaluation/big_2/output/REG")
f= "/home/laskama/Dropbox/evaluation/UJIndoorLoc/b0/f0/final/output/REG_1"
reg: Pipeline = Storable.load(f)#dir_p + "REG_2l_1") # Storable.load("../evaluation/simulation/evaluation/big_2/output/" + "REG") #Storable.load("../evaluation/simulation/evaluation/big_2/output/REG")
#knn: Pipeline = Storable.load(dir_p + "KNN")
......@@ -64,10 +68,10 @@ def main(dir_p="../evaluation/lohan/evaluation/new_knn/output/", f_idx=0):
fp = FloorPlanPlotRec((200, 80), 2, floorplan_bg_img=img)
fp.draw_points(bbox.data_provider.labels[train, 0],
bbox.data_provider.labels[train, 1], color='g', alpha=0.5)
#fp.draw_points(bbox.data_provider.labels[test, 0],
# bbox.data_provider.labels[test, 1], color='r',
# alpha=0.5)
#fp.show_plot()
fp.draw_points(bbox.data_provider.labels[test, 0],
bbox.data_provider.labels[test, 1], color='r',
alpha=0.5)
fp.show_plot()
# for each prediction => find bbox of TD with close
SIM_THRESHOLD = 5
......@@ -144,8 +148,18 @@ def main(dir_p="../evaluation/lohan/evaluation/new_knn/output/", f_idx=0):
coords = [(l[0], l[1]) for l in y_true]
geometry = [Point(xy) for xy in zip(coords)]
df = pd.DataFrame(list(zip(y_pred_box[:,0], y_pred_box[:, 1], knn_error_x, knn_error_y, box_size)), columns=['x', 'y', 'error_x', 'error_y', 'box_size'])
df.plot.scatter(x='x', y='y', c='box_size', colormap='viridis')
plt.axis('equal')
fig = plt.figure()
ax = fig.add_subplot(111)
s = ax.scatter(df.x, df.y, c = df.box_size)
#df.plot.scatter(x='x', y='y', c='box_size', colormap='viridis')
#ax = plt.gca()
ax.set_aspect('equal')
divider = make_axes_locatable(ax)
cax = divider.append_axes("right", size="5%", pad=0.1)
cb = plt.colorbar(s, cax=cax)
cb.set_label('Box size [m²]')
#plt.ylim(0, 80)
df_2 = pd.DataFrame(list(
zip(y_true[:, 0], y_true[:, 1], knn_error_x, knn_error_y,
......@@ -157,6 +171,18 @@ def main(dir_p="../evaluation/lohan/evaluation/new_knn/output/", f_idx=0):
plt.axis('equal')
plt.show()
regr = linear_model.LinearRegression()
# Train the model using the training sets
regr.fit(knn_error.reshape(-1, 1), box_size.reshape(-1, 1))
# Make predictions using the testing set
y_val = regr.predict(knn_error.reshape(-1, 1))
plt.scatter(knn_error, box_size)
plt.plot(knn_error, y_val)
plt.show()
#
# Correlation between std of Monte-Carlo-Dropout Model (center, size, error)
......@@ -197,7 +223,7 @@ def generate_correlation_df():
fold_idx = []
for f_idx in range(1):
summary = main("../evaluation/UJIndoorLoc/b0/f2/overlap/output/", f_idx=f_idx)
summary = main("/home/laskama/Dropbox/evaluation/lohan/", f_idx=f_idx)
y_intercepts += summary["y"]
betas += summary["beta"]
......@@ -215,15 +241,15 @@ def generate_correlation_df():
return df
def gwr(g_coords, g_y, g_X):
gwr_selector = Sel_BW(g_coords, g_y, g_X)
gwr_bw = gwr_selector.search(bw_min=2)
print(gwr_bw)
gwr_results: GWRResults = GWR(g_coords, g_y, g_X, gwr_bw).fit()
print(gwr_results.localR2[0:10])
print(gwr_results.summary())
filter_t = gwr_results.filter_tvals()
# def gwr(g_coords, g_y, g_X):
# gwr_selector = Sel_BW(g_coords, g_y, g_X)
# gwr_bw = gwr_selector.search(bw_min=2)
# print(gwr_bw)
# gwr_results: GWRResults = GWR(g_coords, g_y, g_X, gwr_bw).fit()
# print(gwr_results.localR2[0:10])
# print(gwr_results.summary())
#
# filter_t = gwr_results.filter_tvals()
......
......@@ -212,9 +212,23 @@ def visualize(p_reg: Pipeline=None, p_circle: Pipeline=None, p_quantile: Pipelin
# alpha=0.5)
# fp.show_plot()
#img = "/home/laskama/Dropbox/PhD/BboxPaper/images/blank_bg.png"
#fp_dims = (125, fp_dims[1])
fp = FloorPlanPlotRec(fp_dims, 2, floorplan_bg_img=img)
for idx in range(vis_idx[1]-vis_idx[0]):#len(y_true)):
# plot training data
x_train, y_train = p_box.data_provider.get_train_data(area_labels=False, split_idx=0)
fp.draw_points(y_train[:, 0], y_train[:, 1], color="red", alpha=0.5)
idx_len = vis_idx[1]-vis_idx[0]
sub_sample_mask = np.random.choice(np.arange(vis_idx[0], vis_idx[1]), int(idx_len * 0.7), replace=False)
for idx in range(idx_len):#len(y_true)):
idx += vis_idx[0]
#if idx not in sub_sample_mask:
# continue
#if np.prod(y_pred_box[idx, 2:]) < 600:
# continue
......@@ -228,10 +242,6 @@ def visualize(p_reg: Pipeline=None, p_circle: Pipeline=None, p_quantile: Pipelin
# p_box.data_provider.labels[test, 1], color='r',
# alpha=0.5)
# plot training data
x_train, y_train = p_box.data_provider.get_train_data()
fp.draw_points(y_train[:, 0], y_train[:, 1], color="red", alpha=0.5)
# # find points that are within box
# lab = p_box.data_provider.labels
#
......@@ -342,7 +352,10 @@ if __name__ == "__main__":
f1 = "../evaluation/lohan/evaluation/overlap/output/{}".format("BBOX_aug3")
f2 = "../evaluation/lohan/evaluation/grid_new_loss/output/REG_DEEP"
f1 = "../evaluation/UJIndoorLoc/b0/f2/overlap/output/BBOX_no_aug_4"
f2 = "../evaluation/UJIndoorLoc/b0/f2/overlap/output/REG_DEEP"
f1 = "/home/laskama/Dropbox/evaluation/UJIndoorLoc/b0/f1/final/output/BBOX_1l_5o_aug_5"
f2 = "/home/laskama/Dropbox/evaluation/UJIndoorLoc/b0/f0/final/output/REG_1"
f1 = "/home/laskama/Dropbox/evaluation/lohan/BBOX_1l_3o_aug_1"
f1 = "/home/laskama/PycharmProjects/bboxPrediction/evaluation/UJIndoorLoc/b0/f0/final/output/BBOX_1l_20_aug_1"
main(p_box_f=f1,
p_reg_f=f2, vis_idx=(0,77))
\ No newline at end of file
#p_reg_f=f2,
vis_idx=(0,77))
\ No newline at end of file
......@@ -15,7 +15,7 @@ from base.bbox_model_definition import bbox_model_for_generator
from base.hier_model_definition import hier_model_for_generator
from base.custom_loss import bbox_loss, define_bbox_loss, define_circle_loss, \
define_quantile_loss, \
define_single_quantile_loss, define_yolo_loss, define_grid_quantile_loss, define_yolo_loss_tanh
define_single_quantile_loss, define_yolo_loss, define_grid_quantile_loss, define_yolo_loss_tanh, define_yolo_loss_tanh_no_size
class BboxModel(DnnModel):
......@@ -69,7 +69,8 @@ class BboxModel(DnnModel):
elif self.type == "GRID_OVERLAP-BBOX":
if 'loss' in params:
loss_func = define_yolo_loss_tanh(params['loss'])
#loss_func = define_yolo_loss_tanh(params['loss'])
loss_func = define_yolo_loss_tanh_no_size(params['loss'])
params.update({'last_activation': tanh,
'losses': loss_func})
......
......@@ -856,6 +856,119 @@ def define_yolo_loss_tanh(params):
return yolo_loss
def define_yolo_loss_tanh_no_size(params):
def yolo_loss(y_true, y_pred):
loss = tf.zeros_like(y_true[:, 0])
# grid cell classification loss
grid_cell_pred = y_pred[:, 4::5]
# scale from [-1,1] to [0,1]
grid_cell_pred = tf.multiply(tf.add(grid_cell_pred, 1.0), 0.5)
# grid cell true
grid_cell_true = tf.one_hot(tf.cast(y_true[:, 2], dtype=tf.int32), depth=tf.shape(grid_cell_pred)[1])
class_loss = tf.reduce_sum(tf.squared_difference(grid_cell_pred, grid_cell_true), axis=1)
if "scale" in params["grid"]:
factor = params["grid"]["scale"]
class_loss = tf.multiply(class_loss, tf.constant(factor))
loss += class_loss
# pred subset
g_idx = y_true[:, 2]
# convert to 3-dim tensor
y_pred_sa = tf.reshape(y_pred, (tf.shape(y_true)[0], tf.shape(grid_cell_pred)[1], 5))
# gather along 2nd dimension (Grids)
grid_sub = tf.gather(y_pred_sa, tf.cast(g_idx,tf.int32), axis=1)#[0,0,:]
# Read dimensions
# (see: https://stackoverflow.com/questions/57387169/
# getting-the-diagonal-elements-of-only-part-of-a-tensor)
s = tf.shape(grid_sub)
# Make indices for gathering
ii, jj = tf.meshgrid(tf.range(s[0]), tf.range(s[2]), indexing='ij')
idx = tf.stack([ii, ii, jj], axis=-1)
# Gather result
y_pred_s = tf.gather_nd(grid_sub, idx)
#
# SCALE tanh box size output to [0,1]
#
zeros = tf.zeros([tf.shape(y_pred_s)[0], 2])
ones = tf.ones([tf.shape(y_pred_s)[0], 2])
scale_add = tf.concat([zeros, ones, tf.zeros([tf.shape(y_pred_s)[0], 1])], axis=1)
y_pred_s = tf.add(y_pred_s, scale_add)
# box loss
box_loss = tf.zeros_like(y_true[:, 0])
box_loss_scale = tf.reduce_sum(tf.multiply(grid_cell_pred, grid_cell_true))
# center loss
c_loss = tf.reduce_sum(tf.square(tf.subtract(y_true[:, :2], y_pred_s[:, :2])),
axis=1)
box_loss += c_loss
# if "size" in params:
# size_loss = tf.reduce_sum(tf.square(y_pred_s[:, 2:]), axis=1)