Skip to content

GitLab

Aufgrund einer Wartung wird GitLab am 19.10. zwischen 8:00 und 9:00 Uhr kurzzeitig nicht zur Verfügung stehen. / Due to maintenance, GitLab will be temporarily unavailable on 19.10. between 8:00 and 9:00 am.

Commit 2da02b6e authored by Evgeny Kusmenko's avatar Evgeny Kusmenko
Browse files

Merge branch 'lmdb_default_names_and_check' into 'master' 

Lmdb default names and check

See merge request !15
parents 5ffb5d70 fad537a7
Pipeline #90976 passed with stages
in 6 minutes and 9 seconds
Hide whitespace changes
Inline Side-by-side
src/main/resources/templates/caffe2/CNNCreator.ftl
View file @ 2da02b6e
......@@ -30,6 +30,13 @@ class ${tc.fileNameWithoutEnding}:
def add_input(self, model, batch_size, db, db_type, device_opts):
with core.DeviceScope(device_opts):
if not os.path.isdir(db):
logging.error("Data loading failure. Directory '" + os.path.abspath(db) + "' does not exist.")
sys.exit(1)
elif not (os.path.isfile(os.path.join(db, 'data.mdb')) and os.path.isfile(os.path.join(db, 'lock.mdb'))):
logging.error("Data loading failure. Directory '" + os.path.abspath(db) + "' does not contain lmdb files.")
sys.exit(1)
# load the data
data_uint8, label = brew.db_input(
model,
......@@ -110,7 +117,7 @@ ${tc.include(tc.architecture.body)}
arg_scope = {"order": "NCHW"}
# == Training model ==
train_model= model_helper.ModelHelper(name="train_net", arg_scope=arg_scope)
data, label, train_dataset_size = self.add_input(train_model, batch_size=batch_size, db=os.path.join(self._data_dir_, 'mnist-train-nchw-lmdb'), db_type='lmdb', device_opts=device_opts)
data, label, train_dataset_size = self.add_input(train_model, batch_size=batch_size, db=os.path.join(self._data_dir_, 'train_lmdb'), db_type='lmdb', device_opts=device_opts)
${tc.join(tc.architectureOutputs, ",", "","")} = self.create_model(train_model, data, device_opts=device_opts)
self.add_training_operators(train_model, ${tc.join(tc.architectureOutputs, ",", "","")}, label, device_opts, opt_type, base_learning_rate, policy, stepsize, epsilon, beta1, beta2, gamma, momentum)
self.add_accuracy(train_model, ${tc.join(tc.architectureOutputs, ",", "","")}, label, device_opts, eval_metric)
......@@ -133,7 +140,7 @@ ${tc.include(tc.architecture.body)}
print("== Running Test model ==")
# == Testing model. ==
test_model= model_helper.ModelHelper(name="test_net", arg_scope=arg_scope, init_params=False)
data, label, test_dataset_size = self.add_input(test_model, batch_size=batch_size, db=os.path.join(self._data_dir_, 'mnist-test-nchw-lmdb'), db_type='lmdb', device_opts=device_opts)
data, label, test_dataset_size = self.add_input(test_model, batch_size=batch_size, db=os.path.join(self._data_dir_, 'test_lmdb'), db_type='lmdb', device_opts=device_opts)
${tc.join(tc.architectureOutputs, ",", "","")} = self.create_model(test_model, data, device_opts=device_opts)
self.add_accuracy(test_model, predictions, label, device_opts, eval_metric)
workspace.RunNetOnce(test_model.param_init_net)
......
src/test/resources/target_code/CNNCreator_Alexnet.py
View file @ 2da02b6e
......@@ -30,6 +30,13 @@ class CNNCreator_Alexnet:
def add_input(self, model, batch_size, db, db_type, device_opts):
with core.DeviceScope(device_opts):
if not os.path.isdir(db):
logging.error("Data loading failure. Directory '" + os.path.abspath(db) + "' does not exist.")
sys.exit(1)
elif not (os.path.isfile(os.path.join(db, 'data.mdb')) and os.path.isfile(os.path.join(db, 'lock.mdb'))):
logging.error("Data loading failure. Directory '" + os.path.abspath(db) + "' does not contain lmdb files.")
sys.exit(1)
# load the data
data_uint8, label = brew.db_input(
model,
......@@ -202,7 +209,7 @@ class CNNCreator_Alexnet:
arg_scope = {"order": "NCHW"}
# == Training model ==
train_model= model_helper.ModelHelper(name="train_net", arg_scope=arg_scope)
data, label, train_dataset_size = self.add_input(train_model, batch_size=batch_size, db=os.path.join(self._data_dir_, 'mnist-train-nchw-lmdb'), db_type='lmdb', device_opts=device_opts)
data, label, train_dataset_size = self.add_input(train_model, batch_size=batch_size, db=os.path.join(self._data_dir_, 'train_lmdb'), db_type='lmdb', device_opts=device_opts)
predictions = self.create_model(train_model, data, device_opts=device_opts)
self.add_training_operators(train_model, predictions, label, device_opts, opt_type, base_learning_rate, policy, stepsize, epsilon, beta1, beta2, gamma, momentum)
self.add_accuracy(train_model, predictions, label, device_opts, eval_metric)
......@@ -225,7 +232,7 @@ class CNNCreator_Alexnet:
print("== Running Test model ==")
# == Testing model. ==
test_model= model_helper.ModelHelper(name="test_net", arg_scope=arg_scope, init_params=False)
data, label, test_dataset_size = self.add_input(test_model, batch_size=batch_size, db=os.path.join(self._data_dir_, 'mnist-test-nchw-lmdb'), db_type='lmdb', device_opts=device_opts)
data, label, test_dataset_size = self.add_input(test_model, batch_size=batch_size, db=os.path.join(self._data_dir_, 'test_lmdb'), db_type='lmdb', device_opts=device_opts)
predictions = self.create_model(test_model, data, device_opts=device_opts)
self.add_accuracy(test_model, predictions, label, device_opts, eval_metric)
workspace.RunNetOnce(test_model.param_init_net)
......
src/test/resources/target_code/CNNCreator_CifarClassifierNetwork.py
View file @ 2da02b6e
......@@ -30,6 +30,13 @@ class CNNCreator_CifarClassifierNetwork:
def add_input(self, model, batch_size, db, db_type, device_opts):
with core.DeviceScope(device_opts):
if not os.path.isdir(db):
logging.error("Data loading failure. Directory '" + os.path.abspath(db) + "' does not exist.")
sys.exit(1)
elif not (os.path.isfile(os.path.join(db, 'data.mdb')) and os.path.isfile(os.path.join(db, 'lock.mdb'))):
logging.error("Data loading failure. Directory '" + os.path.abspath(db) + "' does not contain lmdb files.")
sys.exit(1)
# load the data
data_uint8, label = brew.db_input(
model,
......@@ -287,7 +294,7 @@ class CNNCreator_CifarClassifierNetwork:
arg_scope = {"order": "NCHW"}
# == Training model ==
train_model= model_helper.ModelHelper(name="train_net", arg_scope=arg_scope)
data, label, train_dataset_size = self.add_input(train_model, batch_size=batch_size, db=os.path.join(self._data_dir_, 'mnist-train-nchw-lmdb'), db_type='lmdb', device_opts=device_opts)
data, label, train_dataset_size = self.add_input(train_model, batch_size=batch_size, db=os.path.join(self._data_dir_, 'train_lmdb'), db_type='lmdb', device_opts=device_opts)
softmax = self.create_model(train_model, data, device_opts=device_opts)
self.add_training_operators(train_model, softmax, label, device_opts, opt_type, base_learning_rate, policy, stepsize, epsilon, beta1, beta2, gamma, momentum)
self.add_accuracy(train_model, softmax, label, device_opts, eval_metric)
......@@ -310,7 +317,7 @@ class CNNCreator_CifarClassifierNetwork:
print("== Running Test model ==")
# == Testing model. ==
test_model= model_helper.ModelHelper(name="test_net", arg_scope=arg_scope, init_params=False)
data, label, test_dataset_size = self.add_input(test_model, batch_size=batch_size, db=os.path.join(self._data_dir_, 'mnist-test-nchw-lmdb'), db_type='lmdb', device_opts=device_opts)
data, label, test_dataset_size = self.add_input(test_model, batch_size=batch_size, db=os.path.join(self._data_dir_, 'test_lmdb'), db_type='lmdb', device_opts=device_opts)
softmax = self.create_model(test_model, data, device_opts=device_opts)
self.add_accuracy(test_model, predictions, label, device_opts, eval_metric)
workspace.RunNetOnce(test_model.param_init_net)
......
src/test/resources/target_code/CNNCreator_VGG16.py
View file @ 2da02b6e
......@@ -30,6 +30,13 @@ class CNNCreator_VGG16:
def add_input(self, model, batch_size, db, db_type, device_opts):
with core.DeviceScope(device_opts):
if not os.path.isdir(db):
logging.error("Data loading failure. Directory '" + os.path.abspath(db) + "' does not exist.")
sys.exit(1)
elif not (os.path.isfile(os.path.join(db, 'data.mdb')) and os.path.isfile(os.path.join(db, 'lock.mdb'))):
logging.error("Data loading failure. Directory '" + os.path.abspath(db) + "' does not contain lmdb files.")
sys.exit(1)
# load the data
data_uint8, label = brew.db_input(
model,
......@@ -177,7 +184,7 @@ class CNNCreator_VGG16:
arg_scope = {"order": "NCHW"}
# == Training model ==
train_model= model_helper.ModelHelper(name="train_net", arg_scope=arg_scope)
data, label, train_dataset_size = self.add_input(train_model, batch_size=batch_size, db=os.path.join(self._data_dir_, 'mnist-train-nchw-lmdb'), db_type='lmdb', device_opts=device_opts)
data, label, train_dataset_size = self.add_input(train_model, batch_size=batch_size, db=os.path.join(self._data_dir_, 'train_lmdb'), db_type='lmdb', device_opts=device_opts)
predictions = self.create_model(train_model, data, device_opts=device_opts)
self.add_training_operators(train_model, predictions, label, device_opts, opt_type, base_learning_rate, policy, stepsize, epsilon, beta1, beta2, gamma, momentum)
self.add_accuracy(train_model, predictions, label, device_opts, eval_metric)
......@@ -200,7 +207,7 @@ class CNNCreator_VGG16:
print("== Running Test model ==")
# == Testing model. ==
test_model= model_helper.ModelHelper(name="test_net", arg_scope=arg_scope, init_params=False)
data, label, test_dataset_size = self.add_input(test_model, batch_size=batch_size, db=os.path.join(self._data_dir_, 'mnist-test-nchw-lmdb'), db_type='lmdb', device_opts=device_opts)
data, label, test_dataset_size = self.add_input(test_model, batch_size=batch_size, db=os.path.join(self._data_dir_, 'test_lmdb'), db_type='lmdb', device_opts=device_opts)
predictions = self.create_model(test_model, data, device_opts=device_opts)
self.add_accuracy(test_model, predictions, label, device_opts, eval_metric)
workspace.RunNetOnce(test_model.param_init_net)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment