Skip to content
GitLab
Projects
Groups
Snippets
Help
Loading...
Help
What's new
7
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Sign in
Toggle navigation
Open sidebar
Marius Laska
boxPrediction
Commits
614d1889
Commit
614d1889
authored
Nov 24, 2020
by
Marius Laska
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
added support for full grid uji models
parent
ba51ad0d
Changes
9
Hide whitespace changes
Inline
Side-by-side
Showing
9 changed files
with
256 additions
and
90 deletions
+256
-90
analysis/paper/cdf_reg_vs_bbox_modular(final).py
analysis/paper/cdf_reg_vs_bbox_modular(final).py
+24
-14
base/BboxModel.py
base/BboxModel.py
+7
-3
base/bbox_pipeline.py
base/bbox_pipeline.py
+4
-0
base/data_provider_base_grid.py
base/data_provider_base_grid.py
+3
-0
base/ujiindoorloc_fullgrid_provider.py
base/ujiindoorloc_fullgrid_provider.py
+41
-10
config/gpu_b0_f0.yml
config/gpu_b0_f0.yml
+8
-56
config/gpu_uji_full_grid.yml
config/gpu_uji_full_grid.yml
+65
-0
config/lohan_knn.yml
config/lohan_knn.yml
+93
-0
main.py
main.py
+11
-7
No files found.
analysis/paper/cdf_reg_vs_bbox_modular(final).py
View file @
614d1889
...
...
@@ -17,9 +17,10 @@ from analysis.visualize_learning_progress import \
from
main
import
calc_acc_c
dbscan_split
=
False
dataset
=
"lohan"
def
_calc_reg_stats
(
reg_size_dict
,
pipe_name
,
col_val
,
model_name
,
num_avg
=
10
,
show_plot
=
True
):
num_avg
=
10
#
num_avg = 10
dist_list
=
[]
size_list
=
[]
...
...
@@ -114,6 +115,16 @@ def calc_reg_stats(reg_size_dict, num_layers=3, dataset="lohan", show_plot=True)
_calc_reg_stats
(
reg_size_dict
,
pipe_name
,
col_val
,
model_name
,
num_avg
=
10
,
show_plot
=
True
)
# # kNN
# file = "/home/laskama/PycharmProjects/bboxPrediction/evaluation/uji/gpu/b0/f{}/knn/output/kNN"
# file = "/home/laskama/PycharmProjects/bboxPrediction/evaluation/lohan/knn_random/output/kNN"
# file = "/home/laskama/PycharmProjects/bboxPrediction/evaluation/lohan/knn_dbscan/output/kNN"
# model_name = "kNN -> O"
# pipe_name = file + "_{}"
# col_val = colors[-1]
# _calc_reg_stats(reg_size_dict, pipe_name, col_val, model_name,
# num_avg=1, show_plot=True)
plt
.
legend
(
loc
=
"lower right"
)
...
...
@@ -152,12 +163,11 @@ def _calc_box_stats(reg_size_dict, box_dict, mse_list, pipe_name, model_name, pl
num_add_aug_ratio
=
(
len
(
x
)
-
len
(
x2
))
/
len
(
x2
)
elif
num_add_aug_ratio
==
-
1
and
dataset
==
"lohan"
:
pass
# p.data_provider.current_split_idx = f_idx
# x, _ = p.data_provider.get_augmented_train_data()
# x2, _ = p.data_provider.get_train_data(
# labels=p.data_provider.grid_labels)
# num_add_aug_ratio = (len(x) - len(x2)) / len(x2)
p
.
data_provider
.
current_split_idx
=
f_idx
x
,
_
=
p
.
data_provider
.
get_augmented_train_data
()
x2
,
_
=
p
.
data_provider
.
get_train_data
(
labels
=
p
.
data_provider
.
grid_labels
)
num_add_aug_ratio
=
(
len
(
x
)
-
len
(
x2
))
/
len
(
x2
)
acc_box
,
wrong_mask
,
correct_mask
=
calc_acc_c
(
y_true
,
y_pred
)
...
...
@@ -231,10 +241,10 @@ def calc_box_stats(reg_size_dict, dataset="lohan", show_plot=True, calc_aug_gain
for
l
in
[
1
]:
#, 2]:
for
o
in
beta_range
:
#
if
aug
== 0 and
o == 5.0
:
#
box_dict['beta'] += [o]
#
else:
box_dict
[
'beta'
]
+=
[
o
]
*
2
if
o
==
5.
0
and
dataset
==
"UJI"
:
box_dict
[
'beta'
]
+=
[
o
]
else
:
box_dict
[
'beta'
]
+=
[
o
]
*
2
# DNN
if
dataset
==
"lohan"
:
...
...
@@ -339,7 +349,7 @@ def main(dataset="UJI"):
#for l in [1]:
calc_reg_stats
(
reg_size_dict
,
num_layers
=
3
,
dataset
=
dataset
)
box_df
,
num_add_aug_ratio
=
calc_box_stats
(
reg_size_dict
,
dataset
=
dataset
,
calc_aug_gain
=
Tru
e
)
box_df
,
num_add_aug_ratio
=
calc_box_stats
(
reg_size_dict
,
dataset
=
dataset
,
calc_aug_gain
=
Fals
e
)
plt
.
rcParams
.
update
(
plt
.
rcParamsDefault
)
plt
.
rc
(
"savefig"
,
dpi
=
200
)
...
...
@@ -361,7 +371,7 @@ def rename_files():
a_1
=
"augmentation_1"
#for floor in range(4):
for
file
in
glob
.
glob
(
'/media/laskama/Daten/BBOX/GPU/lohan/gpu/
random
/output/
D
NN-DLB_
hidden_layers_1_
delta_7.5_*'
):
for
file
in
glob
.
glob
(
'/media/laskama/Daten/BBOX/GPU/lohan/gpu/
new_dropout
/output/
2D-C
NN-DLB_delta_7.5_*'
):
old_file
=
file
file
=
file
.
replace
(
"_delta_7.5"
,
""
)
...
...
@@ -389,7 +399,7 @@ if __name__ == "__main__":
res_dict
=
{
"Model"
:
[],
"MSE"
:
[],
"fold"
:
[]}
num_add_aug_ratio
=
[]
dataset
=
"
lohan
"
#
dataset = "
UJI
"
f_idx
=
None
floor
=
None
box_df_concat
=
None
...
...
base/BboxModel.py
View file @
614d1889
...
...
@@ -210,9 +210,13 @@ class BboxModel(DnnModel):
dp
.
floorplan_width
/
dp
.
grid_size
)
if
self
.
type
==
"GRID_OVERLAP-BBOX"
or
self
.
type
==
"GRID_OVERLAP-BBOX_CNN"
or
self
.
type
==
"GRID_OVERLAP-BBOX_2D-CNN"
:
y_cols
=
5
*
(
math
.
ceil
(
dp
.
floorplan_height
/
dp
.
grid_size
)
+
1
)
*
(
math
.
ceil
(
dp
.
floorplan_width
/
dp
.
grid_size
)
+
1
)
num_gs
=
dp
.
get_num_grid_cells
()
if
num_gs
is
None
:
y_cols
=
5
*
(
math
.
ceil
(
dp
.
floorplan_height
/
dp
.
grid_size
)
+
1
)
*
(
math
.
ceil
(
dp
.
floorplan_width
/
dp
.
grid_size
)
+
1
)
else
:
y_cols
=
5
*
num_gs
if
self
.
type
==
"GRID_2L-BBOX"
:
# grid encoding with fixed grid size
...
...
base/bbox_pipeline.py
View file @
614d1889
...
...
@@ -12,6 +12,7 @@ from base.KFoldTrainTestSummary import KFoldTrainTestSummary
from
base.gia_data_provider_grid
import
GiaDataProviderGrid
from
base.lohan_data_provider_grid
import
LohanDataProviderGrid
from
base.ujiindoorloc_provider_grid
import
UJIndoorLocProviderGrid
from
base.ujiindoorloc_fullgrid_provider
import
UJIndoorLocProviderFullGrid
class
BboxPipeline
(
Pipeline
):
...
...
@@ -53,6 +54,9 @@ class BboxPipeline(Pipeline):
data_params
=
data_params
,
pre_params
=
pre_params
)
elif
data_params
[
'provider'
]
==
"UJIndoorLocFullProvider"
:
dp
=
UJIndoorLocProviderFullGrid
()
dp
.
setup_dps
(
data_params
,
pre_params
)
else
:
dp
=
None
...
...
base/data_provider_base_grid.py
View file @
614d1889
...
...
@@ -479,6 +479,9 @@ class DataProviderGridBase(DataProviderBase):
return
x_train
,
y_train
def
get_num_grid_cells
(
self
):
return
None
def
in_mem_data_generator
(
self
,
split_idx
=
None
,
mode
=
'train'
,
area_labels
=
True
,
augmentation
=
None
,
autoencoder
=
False
,
noise_percentage
=
None
,
noisy_labels
=
False
,
grid_labels
=
True
,
model_type
=
"classification"
,
batch_size
=
32
):
if
split_idx
is
None
:
split_idx
=
self
.
current_split_idx
...
...
base/ujiindoorloc_fullgrid_provider.py
View file @
614d1889
...
...
@@ -2,6 +2,7 @@ import math
import
pandas
as
pd
import
numpy
as
np
from
il_pipeline.pipeline
import
Pipeline
from
base.ujiindoorloc_provider_grid
import
UJIndoorLocProviderGrid
...
...
@@ -12,7 +13,10 @@ class UJIndoorLocProviderFullGrid(UJIndoorLocProviderGrid):
super
().
__init__
()
self
.
dps
=
{}
def
setup_dps
(
self
):
def
generate_split_indices
(
self
,
pre_params
=
None
):
pass
def
setup_dps
(
self
,
data_params
,
pre_params
):
idx
=
0
for
index
,
comb
in
self
.
get_bld_floor_combinations
().
iterrows
():
...
...
@@ -20,9 +24,13 @@ class UJIndoorLocProviderFullGrid(UJIndoorLocProviderGrid):
#if idx > 3:
# break
data_params
=
{
"building"
:
comb
[
0
],
"floor"
:
comb
[
1
]}
pre_params
=
{
"standardize"
:
False
}
data_params
[
"building"
]
=
comb
[
0
]
data_params
[
"floor"
]
=
comb
[
1
]
#data_params = {"building": comb[0], "floor": comb[1]}
pre_params
[
"standardize"
]
=
False
dp
=
UJIndoorLocProviderFullGrid
.
setup_data_provider
(
data_params
=
data_params
,
pre_params
=
pre_params
)
self
.
split_ratio
=
dp
.
split_ratio
dp
.
transform_to_2dim_overlapping_grid_encoding
(
grid_size
=
40.0
)
...
...
@@ -73,29 +81,52 @@ class UJIndoorLocProviderFullGrid(UJIndoorLocProviderGrid):
self
.
labels
=
np
.
concatenate
(
labels
,
axis
=
0
)
self
.
grid_labels
=
np
.
concatenate
(
grid_labels
,
axis
=
0
)
self
.
dp_mapping
=
mapping
self
.
splits_indices
=
[(
train_idx
,
test_idx
)]
self
.
get_num_grid_cells
()
# concat and shuffle indices
train
=
np
.
concatenate
(
train_idx
,
axis
=
0
)
test
=
np
.
concatenate
(
test_idx
,
axis
=
0
)
np
.
random
.
shuffle
(
train
)
np
.
random
.
shuffle
(
test
)
self
.
splits_indices
=
[(
train
,
test
)]
self
.
num_splits
=
1
self
.
standardize_data
()
def
decode_grid_labels
(
self
,
grid_labels
):
def
_
decode_grid_labels
(
self
,
grid_labels
,
y_true
):
sub
=
grid_labels
[:,
4
::
5
]
chosen
=
np
.
argmax
(
sub
,
axis
=
1
)
decoded_preds
=
np
.
zeros
((
len
(
grid_labels
),
6
))
# find corresponding dp
for
key
,
mapping
in
self
.
dp_mapping
.
items
():
c_range
=
np
.
logical_and
(
chosen
>=
mapping
[
2
],
chosen
<
mapping
[
3
])
dp_idx
=
np
.
where
(
c_range
)[
0
]
dp
=
self
.
dps
[
key
]
orig_labels
=
self
.
labels
[
dp_idx
,
:]
orig_labels
=
y_true
[
dp_idx
,
:]
decoded_labels
=
convert_from_2dim_overlapping_grid
(
grid_labels
[
dp_idx
,
:],
offset
=
mapping
[
2
],
grid_size
=
dp
.
grid_size
,
padding_ratio
=
0.1
,
height
=
dp
.
floorplan_height
,
width
=
dp
.
floorplan_width
)
print
(
"Enc->DEC: {}"
.
format
(
np
.
array_equal
(
orig_labels
,
decoded_labels
[:,
:
2
])))
decoded_preds
[
dp_idx
,
:
4
]
=
decoded_labels
decoded_preds
[
dp_idx
,
4
:]
=
np
.
array
(
key
)
print
(
"Enc->DEC: {}"
.
format
(
np
.
array_equal
(
orig_labels
,
decoded_labels
[:,
:
2
])))
return
decoded_preds
def
decode_pipe_grid_labels
(
self
,
p
:
Pipeline
):
y_pred_list
=
p
.
summary
.
y_pred
y_true_list
=
p
.
summary
.
y_true
y_pred_list_new
=
[]
for
y_pred
,
y_true
in
zip
(
y_pred_list
,
y_true_list
):
y_pred_list_new
.
append
(
self
.
_decode_grid_labels
(
y_pred
,
y_true
))
p
.
summary
.
y_pred
=
y_pred_list_new
def
get_bld_floor_combinations
(
self
):
...
...
@@ -161,7 +192,7 @@ def convert_from_2dim_overlapping_grid(pred_vals, width, height, offset=0, grid_
def
test_encoding_decoding
():
dp
=
UJIndoorLocProviderFullGrid
()
dp
.
setup_dps
()
dp
.
setup_dps
(
data_params
=
{},
pre_params
=
{}
)
# test prediction that chooses correct grid cell and perfect box center
pred_vals
=
np
.
zeros
((
len
(
dp
.
grid_labels
),
267
*
5
))
...
...
@@ -171,7 +202,7 @@ def test_encoding_decoding():
pred_vals
[
gl_idx
,
int
(
gl
[
2
]
*
5
)]
=
gl
[
0
]
pred_vals
[
gl_idx
,
int
(
gl
[
2
]
*
5
+
1
)]
=
gl
[
1
]
dp
.
decode_grid_labels
(
pred_vals
)
dp
.
_
decode_grid_labels
(
pred_vals
)
if
__name__
==
"__main__"
:
...
...
config/gpu_b0_f0.yml
View file @
614d1889
...
...
@@ -6,7 +6,7 @@ data:
# (train, val, test) test=0.2 => 5 fold # The number of temporal epochs into which the dataset is split
split_ratio
:
[
0.7
,
0.1
,
0.2
]
building
:
0
floor
:
0
floor
:
3
#
# are used when not locally set for pipeline
...
...
@@ -15,7 +15,7 @@ data:
global_params
:
# number of experiment repetitions
repetitions
:
1
0
repetitions
:
1
preprocessing
:
# Whether to standardize the RSS values
...
...
@@ -51,62 +51,14 @@ global_params:
pipelines
:
# - name: CNNLoc
# model_params:
# lr: 0.002
# type: CNNLoc_REG
# autoencoder: train
-
name
:
CNNLoc-DLB
model_params
:
lr
:
0.002
batch_size
:
32
augmentation
:
0
autoencoder
:
train
loss
:
grid
:
scale
:
100.0
outside
:
scale
:
1.0
delta
:
[
10.0
,
15.0
,
20.0
]
# - name: DNN
# model_params:
# type: DNN # (DNN, CNN, kNN, SVM) supported (require different parameters)
# pred: regression
# first_neuron: 512
# hidden_layers: [1,2,3]
# lr: 0.7
# batch_size: 32
# #epochs: 200
# dropout: 0.5
# regularization_penalty: 0.0
# augmentation: 0
-
name
:
DNN-DLB
-
name
:
kNN
model_params
:
type
:
GRID_OVERLAP-BBOX
# (DNN, CNN, kNN, SVM) supported (require different parameters)
first_neuron
:
512
hidden_layers
:
[
1
,
2
]
lr
:
0.7
batch_size
:
32
#epochs: 200
dropout
:
0.5
regularization_penalty
:
0.0
augmentation
:
0
loss
:
grid
:
scale
:
100.0
outside
:
scale
:
1.0
delta
:
[
10.0
,
15.0
,
20.0
]
type
:
kNN
k
:
3
pred
:
regression
# base directories for file storage
output
:
model_dir
:
evaluation/uji/gpu/b0/f
0/v2
/output/
summary_dir
:
evaluation/uji/gpu/b0/f
0/v2
/summary/
model_dir
:
evaluation/uji/gpu/b0/f
3/knn
/output/
summary_dir
:
evaluation/uji/gpu/b0/f
3/knn
/summary/
img_folder
:
evaluation/uji/
# folder where floorplan images is located (if not present, will be downloaded)
config/gpu_uji_full_grid.yml
0 → 100644
View file @
614d1889
data
:
# The data provider which should be used
provider
:
UJIndoorLocFullProvider
# File name of floor plan img
floor_plan_img
:
<test>.jpg
# (train, val, test) test=0.2 => 5 fold # The number of temporal epochs into which the dataset is split
split_ratio
:
[
0.7
,
0.1
,
0.2
]
#
# are used when not locally set for pipeline
#
global_params
:
# number of experiment repetitions
repetitions
:
1
preprocessing
:
# Whether to standardize the RSS values
standardize
:
True
# Whether to assign labels with no matching area to closet area
assign_closest
:
False
# The floor number of the Lohan dataset, which should be used
#floor: 0
# The epoch number of the split dataset
#num_epochs: 10
#epoch: 5
# How to check for area matches of labels (to label positions with matching areas)
area_assignment
:
convex_hull
grid_size
:
40
floor_plan
:
# 'segmentation' => computes floor plan segmentation,
# 'regression' => uses DBSCAN to partitions labels into train, test split
type
:
floor_classification
model_params
:
type
:
GRID_OVERLAP-BBOX_CNN
# (DNN, CNN, kNN, SVM) supported (require different parameters)
lr
:
0.7
batch_size
:
32
epochs
:
10
dropout
:
0.5
regularization_penalty
:
0.0
pipelines
:
-
name
:
DNN-DLB
model_params
:
type
:
GRID_OVERLAP-BBOX
# (DNN, CNN, kNN, SVM) supported (require different parameters)
first_neuron
:
512
hidden_layers
:
1
augmentation
:
0
loss
:
grid
:
scale
:
20.0
outside
:
scale
:
1.0
delta
:
7.5
# base directories for file storage
output
:
model_dir
:
evaluation/uji/gpu/full_grid/output/
summary_dir
:
evaluation/uji/gpu/full_grid/summary/
img_folder
:
evaluation/uji/
# folder where floorplan images is located (if not present, will be downloaded)
config/lohan_knn.yml
0 → 100644
View file @
614d1889
data
:
# The data provider which should be used
provider
:
LohanDSprovider
# File name of floor plan img
floor_plan_img
:
<image_name>
# (train, val, test) test=0.2 => 5 fold # The number of temporal epochs into which the dataset is split
split_ratio
:
[
0.7
,
0.1
,
0.2
]
#
# are used when not locally set for pipeline
#
global_params
:
# number of experiment repetitions
repetitions
:
1
preprocessing
:
# Whether to standardize the RSS values
standardize
:
True
# Whether to assign labels with no matching area to closet area
assign_closest
:
False
# The floor number of the Lohan dataset, which should be used
floor
:
0
# The epoch number of the split dataset
#num_epochs: 10
#epoch: 5
# How to check for area matches of labels (to label positions with matching areas)
area_assignment
:
convex_hull
grid_size
:
40
floor_plan
:
# 'segmentation' => computes floor plan segmentation,
# 'regression' => uses DBSCAN to partitions labels into train, test split
type
:
regression
# file where segmentation is stored or loaded from
seg_file
:
evaluation/lohan_reg_seg
# whether to 'load' or compute and 'store' the segmentation
mode
:
load
# (load/store)
# file location that holds numpy array with walls (rows of form [s_x, s_y, e_x, e_y]
walls_file
:
lohan/LohanWalls1stfloor.npy
# parameters of floor plan segmentation algorithm
segmentation
:
# algorithm identifier (currently only LDCE supported)
type
:
LDCE
# file location of precomputed distance file (will be recomputed if it does not exist yet)
precomp_dist_file
:
evaluation/precomp_dist_lohan(0,0).npy
# Parameters of LDCE algorithm
stop_size
:
1
eps
:
0.8
max_eps
:
50
min_pts
:
2
min_members
:
1
rss_penalty
:
0
wall_penalty
:
0
cluster_penalty
:
20
area_mode
:
convex_hull
model_params
:
type
:
GRID_OVERLAP-BBOX
# (DNN, CNN, kNN, SVM) supported (require different parameters)
first_neuron
:
512
hidden_layers
:
1
lr
:
0.7
batch_size
:
32
epochs
:
200
dropout
:
0.5
regularization_penalty
:
0.0
augmentation
:
0
loss
:
grid
:
scale
:
20.0
outside
:
scale
:
1.0
delta
:
20.0
#
# List of pipelines that are executed
#
pipelines
:
-
name
:
kNN
model_params
:
type
:
kNN
k
:
3
pred
:
regression
# base directories for file storage
output
:
model_dir
:
evaluation/lohan/knn_random/output/
summary_dir
:
evaluation/lohan/knn_random/summary/
img_folder
:
evaluation/lohan/
# folder where floorplan images is located (if not present, will be downloaded)
\ No newline at end of file
main.py
View file @
614d1889
...
...
@@ -4,8 +4,11 @@ import logging
from
debug_tools.logger
import
getLogger
from
il_pipeline.utility.config_reader
import
ConfigReader
from
il_pipeline.utility.storable
import
Storable
from
ldce.base
import
ClusterBase
from
base.ujiindoorloc_fullgrid_provider
import
test_encoding_decoding
log
=
getLogger
(
level
=
logging
.
INFO
)
from
base.bbox_pipeline
import
BboxPipeline
import
numpy
as
np
...
...
@@ -117,14 +120,14 @@ def execute(conf_file):
continue
elif
fp_params
[
'type'
]
==
'regression'
:
#
pass
data_provider
.
generate_random_splits_from_clusters
(
clusterer
.
base_cluster_mappings
)
pass
#
data_provider.generate_random_splits_from_clusters(
#
clusterer.base_cluster_mappings)
elif
fp_params
[
'type'
]
==
'floor_classification'
:
data_provider
.
set_area_labels
(
data_provider
.
labels
,
delete_uncovered
=
True
,
pre_params
=
pre_params
)
delete_uncovered
=
True
,
pre_params
=
pre_params
)
# if model is meant for upload use all data for training
if
'upload_split'
in
pre_params
and
pre_params
[
'upload_split'
]:
...
...
@@ -139,7 +142,8 @@ def execute(conf_file):
if
"grid_size"
in
pre_params
:
data_provider
.
transform_to_2dim_3layer_grid_encoding
(
grid_size
=
pre_params
[
'grid_size'
])
elif
"GRID_OVERLAP"
in
model_params
[
'type'
]
or
"GRID_ROT"
in
model_params
[
'type'
]:
elif
(
"GRID_OVERLAP"
in
model_params
[
'type'
]
or
"GRID_ROT"
in
model_params
[
'type'
])
\
and
data_params
[
'provider'
]
!=
"UJIndoorLocFullProvider"
:
if
"grid_size"
in
pre_params
:
data_provider
.
transform_to_2dim_overlapping_grid_encoding
(
grid_size
=
pre_params
[
'grid_size'
])
...
...
@@ -157,7 +161,7 @@ def execute(conf_file):
# angle_only=True, overlap_strategy='keep',
# walls=walls_v)
elif
"GRID"
in
model_params
[
'type'
]:
elif
"GRID"
in
model_params
[
'type'
]
and
data_params
[
'provider'
]
!=
"UJIndoorLocFullProvider"
:
if
"grid_size"
in
pre_params
:
data_provider
.
transform_to_2dim_grid_encoding
(
grid_size
=
pre_params
[
'grid_size'
])
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment