Commit c9b662ef authored by Marius Laska's avatar Marius Laska
Browse files

added new plots for paper

parent fff9845c
......@@ -17,7 +17,7 @@ f_idx = None
def main(res_dict):
fig_title = ""# "Lohan (fold-{})".format(f_idx + 1)
fig_title = "UJIndoorLoc (floor-{})".format(floor) #"Lohan (avg)" #Lohan (fold-{})".format(f_idx + 1)
plt.figure()
plt.title(fig_title)#, floor: {}".format(floor))
plt.xlabel("Area ["+r'$m^2$' + "]")
......@@ -176,9 +176,12 @@ def main(res_dict):
f1 = "/home/laskama/Dropbox/evaluation/UJIndoorLoc/b0/f{}/final/output/BBOX_{}l_{}o{}_{}"
f1 = "/home/laskama/PycharmProjects/bboxPrediction/evaluation/UJIndoorLoc/b0/f{}/final/output/BBOX_{}l_{}{}_{}"
f1 = "/media/laskama/Daten/BBOX/lohan/test/output/BBOX_{}l_{}{}_{}"
f1 = "/media/laskama/Daten/BBOX/UJIndoorLoc/b0/f{}/test/output/BBOX_{}l_{}{}_{}"
mse_list = []
for aug in ["_aug"]:#["_aug", ""]:
for aug in ["", "_aug"]:#["_aug", ""]:
for l in [1,2]:#, 2]:
for o in [5, 10, 15, 20, 25, 30]:#[5, 4, 3]:
......@@ -252,11 +255,12 @@ def main(res_dict):
if __name__ == "__main__":
res_dict = {"Model": [], "MSE": [], "fold": []}
f_idx = None
floor = None
for idx in range(1):
for idx in range(4):
floor = idx
#f_idx = None
#f_idx = idx
res_dict = main(res_dict)
df = pd.DataFrame(res_dict)
......
import numpy as np
from il_pipeline.pipeline import Pipeline
from il_pipeline.utility.storable import Storable
import matplotlib.pyplot as plt
import pandas as pd
import seaborn as sns
from scipy.interpolate import interp1d
from analysis.full_visual_cmp import move_pred_closer
from analysis.plot_analysis import get_avg_box_size
from analysis.visualize_learning_progress import \
convert_from_2dim_overlapping_grid
from main import calc_acc_c
import os
floor = 0
f_idx = None
def main(res_dict):
fig_title = ""#"UJIndoorLoc (floor-{})".format(floor) #"Lohan (avg)" #Lohan (fold-{})".format(f_idx + 1)
plt.figure()
plt.title(fig_title)#, floor: {}".format(floor))
plt.xlabel("Area ["+r'$m^2$' + "]")
plt.ylabel("ACC")
reg_size_dict = {"Model": [], "size": [], "ACC": []}
f2 = "/home/laskama/Dropbox/evaluation/UJIndoorLoc/b0/f{}/final/output/REG_{}"
dist_list = []
size_list = []
mse_list = []
#f2 = "/home/laskama/Dropbox/evaluation/lohan/REG_{}"
for idx in range(10):
p: Pipeline = Storable.load(f2.format(floor, idx+1))
# move_pred_closer(p, dist=2.8)
if f_idx is None:
y_true = np.concatenate(p.summary.y_true_labels, axis=0)
y_pred = np.concatenate(p.summary.y_pred, axis=0)
else:
y_true = p.summary.y_true_labels[f_idx]
y_pred = p.summary.y_pred[f_idx]
n = len(y_true)
dist = np.sort(np.linalg.norm(y_pred - y_true, axis=1))
res_dict["Model"] += ["DNN (3L)"] * len(dist)
res_dict["MSE"] += dist.tolist()
res_dict["fold"] += [f_idx] * len(dist)
mse_list.append(np.mean(dist))
size = np.pi * np.square(dist)
acc = [i/n for i in range(n)]
dist_list.append(dist)
size_list.append(size)
# plt.plot(size, acc)
size_avg = np.mean(np.stack(size_list, axis=1), axis=1)
dist_avg = np.mean(np.stack(dist_list, axis=1), axis=1)
size_avg = np.sort(np.concatenate(size_list, axis=0))
acc_list = [i / (n*10) for i in range(n*10)]
reg_size_dict["Model"].append("DNN->O (3L)")
reg_size_dict["size"].append(size_avg)
reg_size_dict["ACC"].append(np.array(acc_list))
#plt.plot(dist_avg, [i / n for i in range(n)])
#plt.xlabel("Error [m]")
#plt.ylabel("CDF")
plt.plot(size_avg, [i / (n*10) for i in range(n*10)], linestyle="dashed", color="green", label="DNN->O (3L)")
#plt.show()
# plt.figure()
# dist_avg = np.mean(np.stack(dist_list, axis=1), axis=1)
# plt.boxplot(dist_avg)
print("REG_MSE: {}".format(np.mean(np.array(mse_list))))
f2 = "/home/laskama/Dropbox/evaluation/UJIndoorLoc/b0/f{}/final/output/REG_1l_{}"
dist_list = []
size_list = []
mse_list = []
# f2 = "/home/laskama/Dropbox/evaluation/lohan/REG_1l_{}"
for idx in range(10):
p: Pipeline = Storable.load(f2.format(floor, idx + 1))
# move_pred_closer(p, dist=2.8)
if f_idx is None:
y_true = np.concatenate(p.summary.y_true_labels, axis=0)
y_pred = np.concatenate(p.summary.y_pred, axis=0)
else:
y_true = p.summary.y_true_labels[f_idx]
y_pred = p.summary.y_pred[f_idx]
n = len(y_true)
dist = np.sort(np.linalg.norm(y_pred - y_true, axis=1))
res_dict["Model"] += ["DNN (1L)"] * len(dist)
res_dict["MSE"] += dist.tolist()
res_dict["fold"] += [f_idx] * len(dist)
mse_list.append(np.mean(dist))
size = np.pi * np.square(dist)
acc = [i / n for i in range(n)]
dist_list.append(dist)
size_list.append(size)
# plt.plot(size, acc)
size_avg = np.mean(np.stack(size_list, axis=1), axis=1)
size_avg = np.sort(np.concatenate(size_list, axis=0))
reg_size_dict["Model"].append("DNN->O (1L)")
reg_size_dict["size"].append(size_avg)
reg_size_dict["ACC"].append(np.array(acc_list))
plt.plot(size_avg, [i / (n*10) for i in range(n*10)], linestyle="dashed",
color="red", label="DNN->O (1L)")
print("REG_MSE: {}".format(np.mean(np.array(mse_list))))
f2 = "/home/laskama/Dropbox/evaluation/UJIndoorLoc/b0/f{}/final/output/REG_2l_{}"
dist_list = []
size_list = []
mse_list = []
#f2 = "/home/laskama/Dropbox/evaluation/lohan/REG_2l_{}"
for idx in range(10):
p: Pipeline = Storable.load(f2.format(floor, idx + 1))
# move_pred_closer(p, dist=2.8)
if f_idx is None:
y_true = np.concatenate(p.summary.y_true_labels, axis=0)
y_pred = np.concatenate(p.summary.y_pred, axis=0)
else:
y_true = p.summary.y_true_labels[f_idx]
y_pred = p.summary.y_pred[f_idx]
n = len(y_true)
dist = np.sort(np.linalg.norm(y_pred - y_true, axis=1))
res_dict["Model"] += ["DNN (2L)"] * len(dist)
res_dict["MSE"] += dist.tolist()
res_dict["fold"] += [f_idx] * len(dist)
mse_list.append(np.mean(dist))
size = np.pi * np.square(dist)
acc = [i / n for i in range(n)]
dist_list.append(dist)
size_list.append(size)
# plt.plot(size, acc)
size_avg = np.mean(np.stack(size_list, axis=1), axis=1)
size_avg = np.sort(np.concatenate(size_list, axis=0))
reg_size_dict["Model"].append("DNN->O (2L)")
reg_size_dict["size"].append(size_avg)
reg_size_dict["ACC"].append(np.array(acc_list))
plt.plot(size_avg, [i / (n*10) for i in range(n*10)], linestyle="dashed",
color="blue", label="DNN->O (2L)")
print("REG_MSE: {}".format(np.mean(np.array(mse_list))))
# compare BBOX values
f1 = "../evaluation/UJIndoorLoc/b0/f2/final/output/BBOX_{}l_{}o{}_{}"
f1 = "/home/laskama/Dropbox/evaluation/lohan/BBOX_{}l_{}o{}_{}"
f1 = "/home/laskama/Dropbox/evaluation/UJIndoorLoc/b0/f{}/final/output/BBOX_{}l_{}o{}_{}"
f1 = "/home/laskama/PycharmProjects/bboxPrediction/evaluation/UJIndoorLoc/b0/f{}/final/output/BBOX_{}l_{}{}_{}"
f1 = "/media/laskama/Daten/BBOX/lohan/test/output/BBOX_{}l_{}{}_{}"
f1 = "/media/laskama/Daten/BBOX/UJIndoorLoc/b0/f{}/test/output/BBOX_{}l_{}{}_{}"
mse_list = []
box_dict = {"Model": [], "beta": [], "fold": [], "ACC": [],
"size": [], "c-error (mean)": [], "c-error (median)": [],
"c-error (min)": [], "c-error (max)": []}
for aug in ["", "_aug"]:#["_aug", ""]:
for l in [1,2]:#, 2]:
for o in [10, 15, 20]:#[5, 10, 15, 20, 25, 30]:#[5, 4, 3]:
b_size_list = []
acc_box_list = []
dist_list = []
for idx in [1,2,3,4,5]:
#file = f1.format(floor, l, o, aug, idx)
file = f1.format(floor, l, o, aug, idx)
if not os.path.exists(file):
print(file)
continue
# print(file)
p: Pipeline = Storable.load(file)
convert_from_2dim_overlapping_grid(p, grid_size=40, quantile=False,
store=True,
average_samples=True)
if f_idx is None:
y_true = np.concatenate(p.summary.y_true_labels, axis=0)
y_pred = np.concatenate(p.summary.y_pred, axis=0)
else:
y_true = p.summary.y_true_labels[f_idx]
y_pred = p.summary.y_pred[f_idx]
acc_box, wrong_mask, correct_mask = calc_acc_c(y_true, y_pred)
b_size = get_avg_box_size(p)
acc_box_list.append(acc_box)
b_size_list.append(b_size)
dist = np.linalg.norm(y_pred[:, :2] - y_true, axis=1)
dist_list.append(dist)
mse_list.append(dist)
model_name = "BBOX ({}L{})".format(l, aug)
res_dict["Model"] += [model_name] * len(dist)
res_dict["MSE"] += dist.tolist()
res_dict["fold"] += [f_idx] * len(dist)
plt_label = ""
if o == 5:
plt_label = model_name
if aug == "_aug":
c = "r"
else:
c = "g"
if l == 1:
m = "v"
elif l == 2:
m = "^"
elif l == 3:
m = ">"
# plt.scatter(b_size, acc_box, color=c, marker=m)
# plot mean of 3 attemps
size_avg = np.mean(np.array(b_size_list))
acc_avg = np.mean(np.array(acc_box_list))
dist_conc = np.concatenate(dist_list, axis=0)
box_dict["Model"].append(model_name)
box_dict["beta"].append(o)
box_dict["fold"].append(f_idx)
box_dict["ACC"].append(acc_avg)
box_dict["size"].append(size_avg)
box_dict["c-error (mean)"].append(np.mean(dist_conc))
box_dict["c-error (median)"].append(np.median(dist_conc))
box_dict["c-error (min)"].append(np.min(dist_conc))
box_dict["c-error (max)"].append(np.max(dist_conc))
plt.scatter(np.mean(np.array(b_size_list)), np.mean(np.array(acc_box_list)), color=c, marker=m, label=plt_label)
box_df = pd.DataFrame(box_dict)
print("BBOX_MSE: {}".format(np.mean(np.array(mse_list))))
box_dict["best_reg"] = []
box_dict["ACC-gain"] = []
box_dict["size-gain"] = []
# calc ACC gain to each DNN
for _, model in box_df.iterrows():
b_acc = model["ACC"]
b_size = model["size"]
min_gain = 100.0
# find corresponding two values in DNN
for reg_model, reg_size, reg_acc in zip(reg_size_dict["Model"], reg_size_dict["size"], reg_size_dict["ACC"]):
# size on x-axis
test = np.argmax(reg_size > b_size)
x = np.array([reg_size[test-1], reg_size[test]])
y = np.array([reg_acc[test-1], reg_acc[test]])
f = interp1d(x, y)
reg_acc_int = f(b_size)
gain = b_acc - reg_acc_int
# acc on x-axis
test = np.argmax(reg_acc > b_acc)
x = np.array([reg_acc[test - 1], reg_acc[test]])
y = np.array([reg_size[test - 1], reg_size[test]])
f = interp1d(x, y)
reg_size_int = f(b_acc)
size_gain = b_size - reg_size_int
if gain < min_gain:
min_gain = gain
min_size_gain = size_gain
min_model = reg_model
box_dict["ACC-gain"].append(min_gain)
box_dict["size-gain"].append(min_size_gain)
box_dict["best_reg"].append(min_model)
box_df = pd.DataFrame(box_dict)
print("ACC-gain: {}".format(box_df["ACC-gain"].mean()))
#plt.ylim(0.4, 1.0)
plt.xlim(0, 600.0)
plt.legend(loc="lower right")
#plt.show()
return res_dict, box_df
#plt.savefig(fig_title + ".pdf")
if __name__ == "__main__":
res_dict = {"Model": [], "MSE": [], "fold": []}
f_idx = None
floor = None
box_df_concat = None
for idx in range(4):
floor = idx
#f_idx = idx
res_dict, box_df = main(res_dict)
if box_df_concat is None:
box_df_concat = box_df
else:
box_df_concat = pd.concat([box_df_concat, box_df], ignore_index=True)
print("SIZE-GAIN (aug): {}".format(box_df_concat[box_df_concat["Model"].str.contains("aug")]["size-gain"].mean()))
print("SIZE-GAIN (no-aug): {}".format(box_df_concat[~box_df_concat["Model"].str.contains("aug")]["size-gain"].mean()))
print("ACC-GAIN (aug): {}".format(box_df_concat[box_df_concat["Model"].str.contains("aug")]["ACC-gain"].mean()))
print("ACC-GAIN (no-aug): {}".format(box_df_concat[~box_df_concat["Model"].str.contains("aug")]["ACC-gain"].mean()))
df = pd.DataFrame(res_dict)
plt.figure()
sns.boxplot("Model", "MSE", data=df, color="skyblue")
plt.show()
\ No newline at end of file
......@@ -20,6 +20,8 @@ from base.floor_plan_plot import FloorPlanPlotRec
def main(dir_p="../evaluation/lohan/evaluation/new_knn/output/", f_idx=0):
f = "/home/laskama/PycharmProjects/bboxPrediction/evaluation/UJIndoorLoc/b0/f0/final/output/BBOX_1l_20_aug_1"
f = "/media/laskama/Daten/BBOX/UJIndoorLoc/b0/f1/test/output/BBOX_1l_20_aug_1"
f = "/media/laskama/Daten/BBOX/lohan/test/output/BBOX_1l_10_aug_1"
bbox: Pipeline = Storable.load(f)#dir_p + "BBOX_1l_3o_aug_1")
convert_from_2dim_overlapping_grid(bbox, grid_size=40, quantile=False,
......@@ -28,7 +30,8 @@ def main(dir_p="../evaluation/lohan/evaluation/new_knn/output/", f_idx=0):
#convert_from_2dim_grid(bbox, grid_size=40, quantile=False, store=True,
# average_samples=True)
f= "/home/laskama/Dropbox/evaluation/UJIndoorLoc/b0/f0/final/output/REG_1"
f= "/home/laskama/Dropbox/evaluation/UJIndoorLoc/b0/f1/final/output/REG_1"
f = "/home/laskama/Dropbox/evaluation/lohan/REG_1"
reg: Pipeline = Storable.load(f)#dir_p + "REG_2l_1") # Storable.load("../evaluation/simulation/evaluation/big_2/output/" + "REG") #Storable.load("../evaluation/simulation/evaluation/big_2/output/REG")
#knn: Pipeline = Storable.load(dir_p + "KNN")
......
......@@ -218,7 +218,7 @@ def visualize(p_reg: Pipeline=None, p_circle: Pipeline=None, p_quantile: Pipelin
# plot training data
x_train, y_train = p_box.data_provider.get_train_data(area_labels=False, split_idx=0)
fp.draw_points(y_train[:, 0], y_train[:, 1], color="red", alpha=0.5)
fp.draw_points(y_train[:, 0], y_train[:, 1], color="green", alpha=0.5)
idx_len = vis_idx[1]-vis_idx[0]
......@@ -226,8 +226,8 @@ def visualize(p_reg: Pipeline=None, p_circle: Pipeline=None, p_quantile: Pipelin
for idx in range(idx_len):#len(y_true)):
idx += vis_idx[0]
#if idx not in sub_sample_mask:
# continue
if idx not in sub_sample_mask:
continue
#if np.prod(y_pred_box[idx, 2:]) < 600:
# continue
......@@ -289,7 +289,7 @@ def visualize(p_reg: Pipeline=None, p_circle: Pipeline=None, p_quantile: Pipelin
fp.draw_rectangles_new(anchors=y_pred_box[idx, :], color='black')
#fp.draw_ellipse(y_pred_box[idx, :2], y_pred_box[idx, 2], y_pred_box[idx, 3], color="r")
fp.draw_points(y_true[idx, 0], y_true[idx, 1], color='g', alpha=0.5)
fp.draw_points(y_true[idx, 0], y_true[idx, 1], color='red', alpha=0.5)
fp.show_plot()
......@@ -356,6 +356,9 @@ if __name__ == "__main__":
f2 = "/home/laskama/Dropbox/evaluation/UJIndoorLoc/b0/f0/final/output/REG_1"
f1 = "/home/laskama/Dropbox/evaluation/lohan/BBOX_1l_3o_aug_1"
f1 = "/home/laskama/PycharmProjects/bboxPrediction/evaluation/UJIndoorLoc/b0/f0/final/output/BBOX_1l_20_aug_1"
f1 = "/media/laskama/Daten/BBOX/UJIndoorLoc/b0/f1/test/output/BBOX_1l_20_aug_1"
f1 = "/media/laskama/Daten/BBOX/lohan/test/output/BBOX_1l_10_aug_1"
main(p_box_f=f1,
#p_reg_f=f2,
vis_idx=(0,77))
\ No newline at end of file
vis_idx=(0,300))
\ No newline at end of file
from il_pipeline.pipeline import Pipeline
from il_pipeline.pipeline import Storable
from pkg_resources import resource_filename
from base.floor_plan_plot import FloorPlanPlotRec
# p: Pipeline = Storable.load("/media/laskama/Daten/BBOX/lohan/test/output/BBOX_1l_10_aug_1")
p: Pipeline = Storable.load("/media/laskama/Daten/BBOX/UJIndoorLoc/b0/f1/test/output/BBOX_1l_20_aug_1")
img = resource_filename('data', 'lohan/CrowdsourcedDS1floor.png')
#img = "/Users/mariuslaska/sciebo/SensorDatenGIA/Gebäude STL/transparent/2130_4.og.png"
fp_dims = (125, 150) # (200, 80)
#fp_dims = (83.32, 17.6) #50.5)
img = "/home/laskama/Dropbox/PhD/BboxPaper/images/blank_bg.png"
#fp_dims = (125, fp_dims[1])
fp = FloorPlanPlotRec(fp_dims, 5, floorplan_bg_img=img)
fp.draw_points(p.data_provider.labels[:, 0], p.data_provider.labels[:, 1], color="grey", alpha=0.5)
fp.draw_measurement_distribution(p.data_provider.labels, alpha=0.5)
fp.show_plot()
# plot_data_heatmap(pipe, floor_plotter=fp)
\ No newline at end of file
import os
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
from il_pipeline.pipeline import Pipeline
from il_pipeline.utility.storable import Storable
from matplotlib.backends.backend_pdf import PdfPages
from scipy.interpolate import interp1d
import seaborn as sns
from analysis.plot_analysis import get_avg_box_size
from analysis.visualize_learning_progress import \
convert_from_2dim_overlapping_grid
from main import calc_acc_c
def calc_reg_stats(reg_size_dict, num_layers=3, dataset="lohan", show_plot=True):
num_avg = 10
dist_list = []
size_list = []
mse_list = []
model_name = "DNN->O ({}L)".format(num_layers)
colors = ["red", "blue", "green"]
col_val = colors[num_layers-1]
if num_layers == 3:
nl = ""
else:
nl = "_{}l".format(num_layers)
for idx in range(num_avg):
if dataset == "lohan":
file = "/home/laskama/Dropbox/evaluation/lohan/REG{}_{}"
fig_title = "Lohan (avg)"#(fold-{})".format(f_idx + 1)
p: Pipeline = Storable.load(file.format(nl, idx + 1))
elif dataset == "UJI":
file = "/home/laskama/Dropbox/evaluation/UJIndoorLoc/b0/f{}/final/output/REG{}_{}"
fig_title = "UJIndoorLoc (floor-{})".format(floor)
p: Pipeline = Storable.load(file.format(floor, nl, idx + 1))
if f_idx is None:
y_true = np.concatenate(p.summary.y_true_labels, axis=0)
y_pred = np.concatenate(p.summary.y_pred, axis=0)
else:
y_true = p.summary.y_true_labels[f_idx]
y_pred = p.summary.y_pred[f_idx]
n = len(y_true)
dist = np.sort(np.linalg.norm(y_pred - y_true, axis=1))
res_dict["Model"] += [model_name] * len(dist)
res_dict["MSE"] += dist.tolist()
res_dict["fold"] += [f_idx] * len(dist)
mse_list.append(np.mean(dist))
size = np.pi * np.square(dist)
dist_list.append(dist)
size_list.append(size)
size_avg = np.sort(np.concatenate(size_list, axis=0))
acc_list = [i / (n * num_avg) for i in range(n * num_avg)]
reg_size_dict["Model"].append(model_name)
reg_size_dict["size"].append(size_avg)
reg_size_dict["ACC"].append(np.array(acc_list))
if show_plot:
plt.title(fig_title)
plt.xlabel("Area [" + r'$m^2$' + "]")
plt.ylabel("ACC")
plt.plot(size_avg, acc_list,
linestyle="dashed", color=col_val, label=model_name)
# print("REG_MSE: {}".format(np.mean(np.array(mse_list))))
def calc_box_stats(reg_size_dict, dataset="lohan", show_plot=True, calc_aug_gain=True):
if dataset == "lohan":
beta_range = [5, 10, 15]
f1 = "/media/laskama/Daten/BBOX/lohan/test/output/BBOX_{}l_{}{}_{}"
elif dataset == "UJI":
beta_range = [10, 15, 20]
f1 = "/media/laskama/Daten/BBOX/UJIndoorLoc/b0/f{}/test/output/BBOX_{}l_{}{}_{}"
mse_list = []
box_dict = {"Model": [], "beta": [], "fold": [], "ACC": [],
"size": [], "c-error (mean)": [], "c-error (median)": [],
"c-error (min)": [], "c-error (max)": []}
num_add_aug_ratio = -1