Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Sign in
Toggle navigation
Menu
Open sidebar
monticore
EmbeddedMontiArc
generators
CNNArch2Caffe2
Commits
2da02b6e
Commit
2da02b6e
authored
Dec 13, 2018
by
Evgeny Kusmenko
Browse files
Merge branch 'lmdb_default_names_and_check' into 'master'
Lmdb default names and check See merge request
!15
parents
5ffb5d70
fad537a7
Pipeline
#90976
passed with stages
in 6 minutes and 9 seconds
Changes
4
Pipelines
1
Hide whitespace changes
Inline
Side-by-side
src/main/resources/templates/caffe2/CNNCreator.ftl
View file @
2da02b6e
...
...
@@ -30,6 +30,13 @@ class ${tc.fileNameWithoutEnding}:
def add_input(self, model, batch_size, db, db_type, device_opts):
with core.DeviceScope(device_opts):
if not os.path.isdir(db):
logging.error("Data loading failure. Directory '" + os.path.abspath(db) + "' does not exist.")
sys.exit(1)
elif not (os.path.isfile(os.path.join(db, 'data.mdb')) and os.path.isfile(os.path.join(db, 'lock.mdb'))):
logging.error("Data loading failure. Directory '" + os.path.abspath(db) + "' does not contain lmdb files.")
sys.exit(1)
# load the data
data_uint8, label = brew.db_input(
model,
...
...
@@ -110,7 +117,7 @@ ${tc.include(tc.architecture.body)}
arg_scope =
{
"order"
:
"NCHW"
}
# == Training model ==
train_model= model_helper.ModelHelper(name="train_net", arg_scope=arg_scope)
data, label, train_dataset_size = self.add_input(train_model, batch_size=batch_size, db=os.path.join(self._data_dir_, '
mnist-train-nchw-
lmdb'), db_type='lmdb', device_opts=device_opts)
data, label, train_dataset_size = self.add_input(train_model, batch_size=batch_size, db=os.path.join(self._data_dir_, '
train_
lmdb'), db_type='lmdb', device_opts=device_opts)
$
{
tc
.join
(
tc
.architectureOutputs
,
","
,
""
,
""
)}
= self.create_model(train_model, data, device_opts=device_opts)
self.add_training_operators(train_model, $
{
tc
.join
(
tc
.architectureOutputs
,
","
,
""
,
""
)}
, label, device_opts, opt_type, base_learning_rate, policy, stepsize, epsilon, beta1, beta2, gamma, momentum)
self.add_accuracy(train_model, $
{
tc
.join
(
tc
.architectureOutputs
,
","
,
""
,
""
)}
, label, device_opts, eval_metric)
...
...
@@ -133,7 +140,7 @@ ${tc.include(tc.architecture.body)}
print("== Running Test model ==")
# == Testing model. ==
test_model= model_helper.ModelHelper(name="test_net", arg_scope=arg_scope, init_params=False)
data, label, test_dataset_size = self.add_input(test_model, batch_size=batch_size, db=os.path.join(self._data_dir_, '
mnist-test-nchw-
lmdb'), db_type='lmdb', device_opts=device_opts)
data, label, test_dataset_size = self.add_input(test_model, batch_size=batch_size, db=os.path.join(self._data_dir_, '
test_
lmdb'), db_type='lmdb', device_opts=device_opts)
$
{
tc
.join
(
tc
.architectureOutputs
,
","
,
""
,
""
)}
= self.create_model(test_model, data, device_opts=device_opts)
self.add_accuracy(test_model, predictions, label, device_opts, eval_metric)
workspace.RunNetOnce(test_model.param_init_net)
...
...
src/test/resources/target_code/CNNCreator_Alexnet.py
View file @
2da02b6e
...
...
@@ -30,6 +30,13 @@ class CNNCreator_Alexnet:
def
add_input
(
self
,
model
,
batch_size
,
db
,
db_type
,
device_opts
):
with
core
.
DeviceScope
(
device_opts
):
if
not
os
.
path
.
isdir
(
db
):
logging
.
error
(
"Data loading failure. Directory '"
+
os
.
path
.
abspath
(
db
)
+
"' does not exist."
)
sys
.
exit
(
1
)
elif
not
(
os
.
path
.
isfile
(
os
.
path
.
join
(
db
,
'data.mdb'
))
and
os
.
path
.
isfile
(
os
.
path
.
join
(
db
,
'lock.mdb'
))):
logging
.
error
(
"Data loading failure. Directory '"
+
os
.
path
.
abspath
(
db
)
+
"' does not contain lmdb files."
)
sys
.
exit
(
1
)
# load the data
data_uint8
,
label
=
brew
.
db_input
(
model
,
...
...
@@ -202,7 +209,7 @@ class CNNCreator_Alexnet:
arg_scope
=
{
"order"
:
"NCHW"
}
# == Training model ==
train_model
=
model_helper
.
ModelHelper
(
name
=
"train_net"
,
arg_scope
=
arg_scope
)
data
,
label
,
train_dataset_size
=
self
.
add_input
(
train_model
,
batch_size
=
batch_size
,
db
=
os
.
path
.
join
(
self
.
_data_dir_
,
'
mnist-train-nchw-
lmdb'
),
db_type
=
'lmdb'
,
device_opts
=
device_opts
)
data
,
label
,
train_dataset_size
=
self
.
add_input
(
train_model
,
batch_size
=
batch_size
,
db
=
os
.
path
.
join
(
self
.
_data_dir_
,
'
train_
lmdb'
),
db_type
=
'lmdb'
,
device_opts
=
device_opts
)
predictions
=
self
.
create_model
(
train_model
,
data
,
device_opts
=
device_opts
)
self
.
add_training_operators
(
train_model
,
predictions
,
label
,
device_opts
,
opt_type
,
base_learning_rate
,
policy
,
stepsize
,
epsilon
,
beta1
,
beta2
,
gamma
,
momentum
)
self
.
add_accuracy
(
train_model
,
predictions
,
label
,
device_opts
,
eval_metric
)
...
...
@@ -225,7 +232,7 @@ class CNNCreator_Alexnet:
print
(
"== Running Test model =="
)
# == Testing model. ==
test_model
=
model_helper
.
ModelHelper
(
name
=
"test_net"
,
arg_scope
=
arg_scope
,
init_params
=
False
)
data
,
label
,
test_dataset_size
=
self
.
add_input
(
test_model
,
batch_size
=
batch_size
,
db
=
os
.
path
.
join
(
self
.
_data_dir_
,
'
mnist-test-nchw-
lmdb'
),
db_type
=
'lmdb'
,
device_opts
=
device_opts
)
data
,
label
,
test_dataset_size
=
self
.
add_input
(
test_model
,
batch_size
=
batch_size
,
db
=
os
.
path
.
join
(
self
.
_data_dir_
,
'
test_
lmdb'
),
db_type
=
'lmdb'
,
device_opts
=
device_opts
)
predictions
=
self
.
create_model
(
test_model
,
data
,
device_opts
=
device_opts
)
self
.
add_accuracy
(
test_model
,
predictions
,
label
,
device_opts
,
eval_metric
)
workspace
.
RunNetOnce
(
test_model
.
param_init_net
)
...
...
src/test/resources/target_code/CNNCreator_CifarClassifierNetwork.py
View file @
2da02b6e
...
...
@@ -30,6 +30,13 @@ class CNNCreator_CifarClassifierNetwork:
def
add_input
(
self
,
model
,
batch_size
,
db
,
db_type
,
device_opts
):
with
core
.
DeviceScope
(
device_opts
):
if
not
os
.
path
.
isdir
(
db
):
logging
.
error
(
"Data loading failure. Directory '"
+
os
.
path
.
abspath
(
db
)
+
"' does not exist."
)
sys
.
exit
(
1
)
elif
not
(
os
.
path
.
isfile
(
os
.
path
.
join
(
db
,
'data.mdb'
))
and
os
.
path
.
isfile
(
os
.
path
.
join
(
db
,
'lock.mdb'
))):
logging
.
error
(
"Data loading failure. Directory '"
+
os
.
path
.
abspath
(
db
)
+
"' does not contain lmdb files."
)
sys
.
exit
(
1
)
# load the data
data_uint8
,
label
=
brew
.
db_input
(
model
,
...
...
@@ -287,7 +294,7 @@ class CNNCreator_CifarClassifierNetwork:
arg_scope
=
{
"order"
:
"NCHW"
}
# == Training model ==
train_model
=
model_helper
.
ModelHelper
(
name
=
"train_net"
,
arg_scope
=
arg_scope
)
data
,
label
,
train_dataset_size
=
self
.
add_input
(
train_model
,
batch_size
=
batch_size
,
db
=
os
.
path
.
join
(
self
.
_data_dir_
,
'
mnist-train-nchw-
lmdb'
),
db_type
=
'lmdb'
,
device_opts
=
device_opts
)
data
,
label
,
train_dataset_size
=
self
.
add_input
(
train_model
,
batch_size
=
batch_size
,
db
=
os
.
path
.
join
(
self
.
_data_dir_
,
'
train_
lmdb'
),
db_type
=
'lmdb'
,
device_opts
=
device_opts
)
softmax
=
self
.
create_model
(
train_model
,
data
,
device_opts
=
device_opts
)
self
.
add_training_operators
(
train_model
,
softmax
,
label
,
device_opts
,
opt_type
,
base_learning_rate
,
policy
,
stepsize
,
epsilon
,
beta1
,
beta2
,
gamma
,
momentum
)
self
.
add_accuracy
(
train_model
,
softmax
,
label
,
device_opts
,
eval_metric
)
...
...
@@ -310,7 +317,7 @@ class CNNCreator_CifarClassifierNetwork:
print
(
"== Running Test model =="
)
# == Testing model. ==
test_model
=
model_helper
.
ModelHelper
(
name
=
"test_net"
,
arg_scope
=
arg_scope
,
init_params
=
False
)
data
,
label
,
test_dataset_size
=
self
.
add_input
(
test_model
,
batch_size
=
batch_size
,
db
=
os
.
path
.
join
(
self
.
_data_dir_
,
'
mnist-test-nchw-
lmdb'
),
db_type
=
'lmdb'
,
device_opts
=
device_opts
)
data
,
label
,
test_dataset_size
=
self
.
add_input
(
test_model
,
batch_size
=
batch_size
,
db
=
os
.
path
.
join
(
self
.
_data_dir_
,
'
test_
lmdb'
),
db_type
=
'lmdb'
,
device_opts
=
device_opts
)
softmax
=
self
.
create_model
(
test_model
,
data
,
device_opts
=
device_opts
)
self
.
add_accuracy
(
test_model
,
predictions
,
label
,
device_opts
,
eval_metric
)
workspace
.
RunNetOnce
(
test_model
.
param_init_net
)
...
...
src/test/resources/target_code/CNNCreator_VGG16.py
View file @
2da02b6e
...
...
@@ -30,6 +30,13 @@ class CNNCreator_VGG16:
def
add_input
(
self
,
model
,
batch_size
,
db
,
db_type
,
device_opts
):
with
core
.
DeviceScope
(
device_opts
):
if
not
os
.
path
.
isdir
(
db
):
logging
.
error
(
"Data loading failure. Directory '"
+
os
.
path
.
abspath
(
db
)
+
"' does not exist."
)
sys
.
exit
(
1
)
elif
not
(
os
.
path
.
isfile
(
os
.
path
.
join
(
db
,
'data.mdb'
))
and
os
.
path
.
isfile
(
os
.
path
.
join
(
db
,
'lock.mdb'
))):
logging
.
error
(
"Data loading failure. Directory '"
+
os
.
path
.
abspath
(
db
)
+
"' does not contain lmdb files."
)
sys
.
exit
(
1
)
# load the data
data_uint8
,
label
=
brew
.
db_input
(
model
,
...
...
@@ -177,7 +184,7 @@ class CNNCreator_VGG16:
arg_scope
=
{
"order"
:
"NCHW"
}
# == Training model ==
train_model
=
model_helper
.
ModelHelper
(
name
=
"train_net"
,
arg_scope
=
arg_scope
)
data
,
label
,
train_dataset_size
=
self
.
add_input
(
train_model
,
batch_size
=
batch_size
,
db
=
os
.
path
.
join
(
self
.
_data_dir_
,
'
mnist-train-nchw-
lmdb'
),
db_type
=
'lmdb'
,
device_opts
=
device_opts
)
data
,
label
,
train_dataset_size
=
self
.
add_input
(
train_model
,
batch_size
=
batch_size
,
db
=
os
.
path
.
join
(
self
.
_data_dir_
,
'
train_
lmdb'
),
db_type
=
'lmdb'
,
device_opts
=
device_opts
)
predictions
=
self
.
create_model
(
train_model
,
data
,
device_opts
=
device_opts
)
self
.
add_training_operators
(
train_model
,
predictions
,
label
,
device_opts
,
opt_type
,
base_learning_rate
,
policy
,
stepsize
,
epsilon
,
beta1
,
beta2
,
gamma
,
momentum
)
self
.
add_accuracy
(
train_model
,
predictions
,
label
,
device_opts
,
eval_metric
)
...
...
@@ -200,7 +207,7 @@ class CNNCreator_VGG16:
print
(
"== Running Test model =="
)
# == Testing model. ==
test_model
=
model_helper
.
ModelHelper
(
name
=
"test_net"
,
arg_scope
=
arg_scope
,
init_params
=
False
)
data
,
label
,
test_dataset_size
=
self
.
add_input
(
test_model
,
batch_size
=
batch_size
,
db
=
os
.
path
.
join
(
self
.
_data_dir_
,
'
mnist-test-nchw-
lmdb'
),
db_type
=
'lmdb'
,
device_opts
=
device_opts
)
data
,
label
,
test_dataset_size
=
self
.
add_input
(
test_model
,
batch_size
=
batch_size
,
db
=
os
.
path
.
join
(
self
.
_data_dir_
,
'
test_
lmdb'
),
db_type
=
'lmdb'
,
device_opts
=
device_opts
)
predictions
=
self
.
create_model
(
test_model
,
data
,
device_opts
=
device_opts
)
self
.
add_accuracy
(
test_model
,
predictions
,
label
,
device_opts
,
eval_metric
)
workspace
.
RunNetOnce
(
test_model
.
param_init_net
)
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment