Commit 614d1889 authored by Marius Laska's avatar Marius Laska
Browse files

added support for full grid uji models

parent ba51ad0d
......@@ -17,9 +17,10 @@ from analysis.visualize_learning_progress import \
from main import calc_acc_c
dbscan_split = False
dataset = "lohan"
def _calc_reg_stats(reg_size_dict, pipe_name, col_val, model_name, num_avg= 10, show_plot=True):
num_avg = 10
#num_avg = 10
dist_list = []
size_list = []
......@@ -114,6 +115,16 @@ def calc_reg_stats(reg_size_dict, num_layers=3, dataset="lohan", show_plot=True)
_calc_reg_stats(reg_size_dict, pipe_name, col_val, model_name,
num_avg=10, show_plot=True)
# # kNN
# file = "/home/laskama/PycharmProjects/bboxPrediction/evaluation/uji/gpu/b0/f{}/knn/output/kNN"
# file = "/home/laskama/PycharmProjects/bboxPrediction/evaluation/lohan/knn_random/output/kNN"
# file = "/home/laskama/PycharmProjects/bboxPrediction/evaluation/lohan/knn_dbscan/output/kNN"
# model_name = "kNN -> O"
# pipe_name = file + "_{}"
# col_val = colors[-1]
# _calc_reg_stats(reg_size_dict, pipe_name, col_val, model_name,
# num_avg=1, show_plot=True)
plt.legend(loc="lower right")
......@@ -152,12 +163,11 @@ def _calc_box_stats(reg_size_dict, box_dict, mse_list, pipe_name, model_name, pl
num_add_aug_ratio = (len(x) - len(x2)) / len(x2)
elif num_add_aug_ratio == -1 and dataset == "lohan":
pass
# p.data_provider.current_split_idx = f_idx
# x, _ = p.data_provider.get_augmented_train_data()
# x2, _ = p.data_provider.get_train_data(
# labels=p.data_provider.grid_labels)
# num_add_aug_ratio = (len(x) - len(x2)) / len(x2)
p.data_provider.current_split_idx = f_idx
x, _ = p.data_provider.get_augmented_train_data()
x2, _ = p.data_provider.get_train_data(
labels=p.data_provider.grid_labels)
num_add_aug_ratio = (len(x) - len(x2)) / len(x2)
acc_box, wrong_mask, correct_mask = calc_acc_c(y_true,
y_pred)
......@@ -231,10 +241,10 @@ def calc_box_stats(reg_size_dict, dataset="lohan", show_plot=True, calc_aug_gain
for l in [1]:#, 2]:
for o in beta_range:
#if aug == 0 and o == 5.0:
# box_dict['beta'] += [o]
#else:
box_dict['beta'] += [o] * 2
if o == 5.0 and dataset == "UJI":
box_dict['beta'] += [o]
else:
box_dict['beta'] += [o] * 2
# DNN
if dataset == "lohan":
......@@ -339,7 +349,7 @@ def main(dataset="UJI"):
#for l in [1]:
calc_reg_stats(reg_size_dict, num_layers=3, dataset=dataset)
box_df, num_add_aug_ratio = calc_box_stats(reg_size_dict, dataset=dataset, calc_aug_gain=True)
box_df, num_add_aug_ratio = calc_box_stats(reg_size_dict, dataset=dataset, calc_aug_gain=False)
plt.rcParams.update(plt.rcParamsDefault)
plt.rc("savefig", dpi=200)
......@@ -361,7 +371,7 @@ def rename_files():
a_1 = "augmentation_1"
#for floor in range(4):
for file in glob.glob('/media/laskama/Daten/BBOX/GPU/lohan/gpu/random/output/DNN-DLB_hidden_layers_1_delta_7.5_*'):
for file in glob.glob('/media/laskama/Daten/BBOX/GPU/lohan/gpu/new_dropout/output/2D-CNN-DLB_delta_7.5_*'):
old_file = file
file = file.replace("_delta_7.5", "")
......@@ -389,7 +399,7 @@ if __name__ == "__main__":
res_dict = {"Model": [], "MSE": [], "fold": []}
num_add_aug_ratio = []
dataset = "lohan"
# dataset = "UJI"
f_idx = None
floor = None
box_df_concat = None
......
......@@ -210,9 +210,13 @@ class BboxModel(DnnModel):
dp.floorplan_width / dp.grid_size)
if self.type == "GRID_OVERLAP-BBOX" or self.type == "GRID_OVERLAP-BBOX_CNN" or self.type == "GRID_OVERLAP-BBOX_2D-CNN":
y_cols = 5 * (math.ceil(
dp.floorplan_height / dp.grid_size) + 1) * (math.ceil(
dp.floorplan_width / dp.grid_size) + 1)
num_gs = dp.get_num_grid_cells()
if num_gs is None:
y_cols = 5 * (math.ceil(
dp.floorplan_height / dp.grid_size) + 1) * (math.ceil(
dp.floorplan_width / dp.grid_size) + 1)
else:
y_cols = 5 * num_gs
if self.type == "GRID_2L-BBOX":
# grid encoding with fixed grid size
......
......@@ -12,6 +12,7 @@ from base.KFoldTrainTestSummary import KFoldTrainTestSummary
from base.gia_data_provider_grid import GiaDataProviderGrid
from base.lohan_data_provider_grid import LohanDataProviderGrid
from base.ujiindoorloc_provider_grid import UJIndoorLocProviderGrid
from base.ujiindoorloc_fullgrid_provider import UJIndoorLocProviderFullGrid
class BboxPipeline(Pipeline):
......@@ -53,6 +54,9 @@ class BboxPipeline(Pipeline):
data_params=data_params,
pre_params=pre_params
)
elif data_params['provider'] == "UJIndoorLocFullProvider":
dp = UJIndoorLocProviderFullGrid()
dp.setup_dps(data_params, pre_params)
else:
dp = None
......
......@@ -479,6 +479,9 @@ class DataProviderGridBase(DataProviderBase):
return x_train, y_train
def get_num_grid_cells(self):
return None
def in_mem_data_generator(self, split_idx=None, mode='train', area_labels=True, augmentation=None, autoencoder=False, noise_percentage=None, noisy_labels=False, grid_labels=True, model_type="classification", batch_size=32):
if split_idx is None:
split_idx = self.current_split_idx
......
......@@ -2,6 +2,7 @@ import math
import pandas as pd
import numpy as np
from il_pipeline.pipeline import Pipeline
from base.ujiindoorloc_provider_grid import UJIndoorLocProviderGrid
......@@ -12,7 +13,10 @@ class UJIndoorLocProviderFullGrid(UJIndoorLocProviderGrid):
super().__init__()
self.dps = {}
def setup_dps(self):
def generate_split_indices(self, pre_params=None):
pass
def setup_dps(self, data_params, pre_params):
idx = 0
for index, comb in self.get_bld_floor_combinations().iterrows():
......@@ -20,9 +24,13 @@ class UJIndoorLocProviderFullGrid(UJIndoorLocProviderGrid):
#if idx > 3:
# break
data_params = {"building": comb[0], "floor": comb[1]}
pre_params = {"standardize": False}
data_params["building"] = comb[0]
data_params["floor"] = comb[1]
#data_params = {"building": comb[0], "floor": comb[1]}
pre_params["standardize"] = False
dp = UJIndoorLocProviderFullGrid.setup_data_provider(data_params=data_params, pre_params=pre_params)
self.split_ratio = dp.split_ratio
dp.transform_to_2dim_overlapping_grid_encoding(
grid_size=40.0)
......@@ -73,29 +81,52 @@ class UJIndoorLocProviderFullGrid(UJIndoorLocProviderGrid):
self.labels = np.concatenate(labels, axis=0)
self.grid_labels = np.concatenate(grid_labels, axis=0)
self.dp_mapping = mapping
self.splits_indices = [(train_idx, test_idx)]
self.get_num_grid_cells()
# concat and shuffle indices
train = np.concatenate(train_idx, axis=0)
test = np.concatenate(test_idx, axis=0)
np.random.shuffle(train)
np.random.shuffle(test)
self.splits_indices = [(train, test)]
self.num_splits = 1
self.standardize_data()
def decode_grid_labels(self, grid_labels):
def _decode_grid_labels(self, grid_labels, y_true):
sub = grid_labels[:, 4::5]
chosen = np.argmax(sub, axis=1)
decoded_preds = np.zeros((len(grid_labels), 6))
# find corresponding dp
for key, mapping in self.dp_mapping.items():
c_range = np.logical_and(chosen >= mapping[2], chosen < mapping[3])
dp_idx = np.where(c_range)[0]
dp = self.dps[key]
orig_labels = self.labels[dp_idx, :]
orig_labels = y_true[dp_idx, :]
decoded_labels = convert_from_2dim_overlapping_grid(
grid_labels[dp_idx, :], offset=mapping[2],
grid_size=dp.grid_size, padding_ratio=0.1,
height=dp.floorplan_height, width=dp.floorplan_width)
print("Enc->DEC: {}".format(np.array_equal(orig_labels, decoded_labels[:, :2])))
decoded_preds[dp_idx, :4] = decoded_labels
decoded_preds[dp_idx, 4:] = np.array(key)
print("Enc->DEC: {}".format(
np.array_equal(orig_labels, decoded_labels[:, :2])))
return decoded_preds
def decode_pipe_grid_labels(self, p: Pipeline):
y_pred_list = p.summary.y_pred
y_true_list = p.summary.y_true
y_pred_list_new = []
for y_pred, y_true in zip(y_pred_list, y_true_list):
y_pred_list_new.append(self._decode_grid_labels(y_pred, y_true))
p.summary.y_pred = y_pred_list_new
def get_bld_floor_combinations(self):
......@@ -161,7 +192,7 @@ def convert_from_2dim_overlapping_grid(pred_vals, width, height, offset=0, grid_
def test_encoding_decoding():
dp = UJIndoorLocProviderFullGrid()
dp.setup_dps()
dp.setup_dps(data_params={}, pre_params={})
# test prediction that chooses correct grid cell and perfect box center
pred_vals = np.zeros((len(dp.grid_labels), 267 * 5))
......@@ -171,7 +202,7 @@ def test_encoding_decoding():
pred_vals[gl_idx, int(gl[2] * 5)] = gl[0]
pred_vals[gl_idx, int(gl[2] * 5 + 1)] = gl[1]
dp.decode_grid_labels(pred_vals)
dp._decode_grid_labels(pred_vals)
if __name__ == "__main__":
......
......@@ -6,7 +6,7 @@ data:
# (train, val, test) test=0.2 => 5 fold # The number of temporal epochs into which the dataset is split
split_ratio: [0.7, 0.1, 0.2]
building: 0
floor: 0
floor: 3
#
# are used when not locally set for pipeline
......@@ -15,7 +15,7 @@ data:
global_params:
# number of experiment repetitions
repetitions: 10
repetitions: 1
preprocessing:
# Whether to standardize the RSS values
......@@ -51,62 +51,14 @@ global_params:
pipelines:
# - name: CNNLoc
# model_params:
# lr: 0.002
# type: CNNLoc_REG
# autoencoder: train
- name: CNNLoc-DLB
model_params:
lr: 0.002
batch_size: 32
augmentation: 0
autoencoder: train
loss:
grid:
scale: 100.0
outside:
scale: 1.0
delta: [10.0, 15.0, 20.0]
# - name: DNN
# model_params:
# type: DNN # (DNN, CNN, kNN, SVM) supported (require different parameters)
# pred: regression
# first_neuron: 512
# hidden_layers: [1,2,3]
# lr: 0.7
# batch_size: 32
# #epochs: 200
# dropout: 0.5
# regularization_penalty: 0.0
# augmentation: 0
- name: DNN-DLB
- name: kNN
model_params:
type: GRID_OVERLAP-BBOX # (DNN, CNN, kNN, SVM) supported (require different parameters)
first_neuron: 512
hidden_layers: [1,2]
lr: 0.7
batch_size: 32
#epochs: 200
dropout: 0.5
regularization_penalty: 0.0
augmentation: 0
loss:
grid:
scale: 100.0
outside:
scale: 1.0
delta: [10.0, 15.0, 20.0]
type: kNN
k: 3
pred: regression
# base directories for file storage
output:
model_dir: evaluation/uji/gpu/b0/f0/v2/output/
summary_dir: evaluation/uji/gpu/b0/f0/v2/summary/
model_dir: evaluation/uji/gpu/b0/f3/knn/output/
summary_dir: evaluation/uji/gpu/b0/f3/knn/summary/
img_folder: evaluation/uji/ # folder where floorplan images is located (if not present, will be downloaded)
data:
# The data provider which should be used
provider: UJIndoorLocFullProvider
# File name of floor plan img
floor_plan_img: <test>.jpg
# (train, val, test) test=0.2 => 5 fold # The number of temporal epochs into which the dataset is split
split_ratio: [0.7, 0.1, 0.2]
#
# are used when not locally set for pipeline
#
global_params:
# number of experiment repetitions
repetitions: 1
preprocessing:
# Whether to standardize the RSS values
standardize: True
# Whether to assign labels with no matching area to closet area
assign_closest: False
# The floor number of the Lohan dataset, which should be used
#floor: 0
# The epoch number of the split dataset
#num_epochs: 10
#epoch: 5
# How to check for area matches of labels (to label positions with matching areas)
area_assignment: convex_hull
grid_size: 40
floor_plan:
# 'segmentation' => computes floor plan segmentation,
# 'regression' => uses DBSCAN to partitions labels into train, test split
type: floor_classification
model_params:
type: GRID_OVERLAP-BBOX_CNN # (DNN, CNN, kNN, SVM) supported (require different parameters)
lr: 0.7
batch_size: 32
epochs: 10
dropout: 0.5
regularization_penalty: 0.0
pipelines:
- name: DNN-DLB
model_params:
type: GRID_OVERLAP-BBOX # (DNN, CNN, kNN, SVM) supported (require different parameters)
first_neuron: 512
hidden_layers: 1
augmentation: 0
loss:
grid:
scale: 20.0
outside:
scale: 1.0
delta: 7.5
# base directories for file storage
output:
model_dir: evaluation/uji/gpu/full_grid/output/
summary_dir: evaluation/uji/gpu/full_grid/summary/
img_folder: evaluation/uji/ # folder where floorplan images is located (if not present, will be downloaded)
data:
# The data provider which should be used
provider: LohanDSprovider
# File name of floor plan img
floor_plan_img: <image_name>
# (train, val, test) test=0.2 => 5 fold # The number of temporal epochs into which the dataset is split
split_ratio: [0.7, 0.1, 0.2]
#
# are used when not locally set for pipeline
#
global_params:
# number of experiment repetitions
repetitions: 1
preprocessing:
# Whether to standardize the RSS values
standardize: True
# Whether to assign labels with no matching area to closet area
assign_closest: False
# The floor number of the Lohan dataset, which should be used
floor: 0
# The epoch number of the split dataset
#num_epochs: 10
#epoch: 5
# How to check for area matches of labels (to label positions with matching areas)
area_assignment: convex_hull
grid_size: 40
floor_plan:
# 'segmentation' => computes floor plan segmentation,
# 'regression' => uses DBSCAN to partitions labels into train, test split
type: regression
# file where segmentation is stored or loaded from
seg_file: evaluation/lohan_reg_seg
# whether to 'load' or compute and 'store' the segmentation
mode: load # (load/store)
# file location that holds numpy array with walls (rows of form [s_x, s_y, e_x, e_y]
walls_file: lohan/LohanWalls1stfloor.npy
# parameters of floor plan segmentation algorithm
segmentation:
# algorithm identifier (currently only LDCE supported)
type: LDCE
# file location of precomputed distance file (will be recomputed if it does not exist yet)
precomp_dist_file: evaluation/precomp_dist_lohan(0,0).npy
# Parameters of LDCE algorithm
stop_size: 1
eps: 0.8
max_eps: 50
min_pts: 2
min_members: 1
rss_penalty: 0
wall_penalty: 0
cluster_penalty: 20
area_mode: convex_hull
model_params:
type: GRID_OVERLAP-BBOX # (DNN, CNN, kNN, SVM) supported (require different parameters)
first_neuron: 512
hidden_layers: 1
lr: 0.7
batch_size: 32
epochs: 200
dropout: 0.5
regularization_penalty: 0.0
augmentation: 0
loss:
grid:
scale: 20.0
outside:
scale: 1.0
delta: 20.0
#
# List of pipelines that are executed
#
pipelines:
- name: kNN
model_params:
type: kNN
k: 3
pred: regression
# base directories for file storage
output:
model_dir: evaluation/lohan/knn_random/output/
summary_dir: evaluation/lohan/knn_random/summary/
img_folder: evaluation/lohan/ # folder where floorplan images is located (if not present, will be downloaded)
\ No newline at end of file
......@@ -4,8 +4,11 @@ import logging
from debug_tools.logger import getLogger
from il_pipeline.utility.config_reader import ConfigReader
from il_pipeline.utility.storable import Storable
from ldce.base import ClusterBase
from base.ujiindoorloc_fullgrid_provider import test_encoding_decoding
log = getLogger(level=logging.INFO)
from base.bbox_pipeline import BboxPipeline
import numpy as np
......@@ -117,14 +120,14 @@ def execute(conf_file):
continue
elif fp_params['type'] == 'regression':
#pass
data_provider.generate_random_splits_from_clusters(
clusterer.base_cluster_mappings)
pass
#data_provider.generate_random_splits_from_clusters(
# clusterer.base_cluster_mappings)
elif fp_params['type'] == 'floor_classification':
data_provider.set_area_labels(data_provider.labels,
delete_uncovered=True,
pre_params=pre_params)
delete_uncovered=True,
pre_params=pre_params)
# if model is meant for upload use all data for training
if 'upload_split' in pre_params and pre_params['upload_split']:
......@@ -139,7 +142,8 @@ def execute(conf_file):
if "grid_size" in pre_params:
data_provider.transform_to_2dim_3layer_grid_encoding(
grid_size=pre_params['grid_size'])
elif "GRID_OVERLAP" in model_params['type'] or "GRID_ROT" in model_params['type']:
elif ("GRID_OVERLAP" in model_params['type'] or "GRID_ROT" in model_params['type']) \
and data_params['provider'] != "UJIndoorLocFullProvider":
if "grid_size" in pre_params:
data_provider.transform_to_2dim_overlapping_grid_encoding(
grid_size=pre_params['grid_size'])
......@@ -157,7 +161,7 @@ def execute(conf_file):
# angle_only=True, overlap_strategy='keep',
# walls=walls_v)
elif "GRID" in model_params['type']:
elif "GRID" in model_params['type'] and data_params['provider'] != "UJIndoorLocFullProvider":
if "grid_size" in pre_params:
data_provider.transform_to_2dim_grid_encoding(
grid_size=pre_params['grid_size'])
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment