Added time measurements for training and testing, restricted use of accuracy...

Added time measurements for training and testing, restricted use of accuracy for losses different than euclidean loss since it is not supported by the euclidean loss in Caffe2
parent 36ef58b9
......@@ -3,6 +3,7 @@ from caffe2.python.predictor import mobile_exporter
from caffe2.proto import caffe2_pb2
import numpy as np
import math
import datetime
import logging
import os
import sys
......@@ -28,6 +29,15 @@ class ${tc.fileNameWithoutEnding}:
return iterations_int
def get_epoch_as_iter(self, num_epoch, batch_size, dataset_size): #To print metric durint training process
#Force floating point calculation
batch_size_float = float(batch_size)
dataset_size_float = float(dataset_size)
epoch_float = math.ceil(dataset_size_float/batch_size_float)
epoch_int = int(epoch_float)
return epoch_int
def add_input(self, model, batch_size, db, db_type, device_opts):
with core.DeviceScope(device_opts):
......@@ -125,7 +135,8 @@ ${tc.include(tc.architecture.body)}
data, label, train_dataset_size = self.add_input(train_model, batch_size=batch_size, db=os.path.join(self._data_dir_, 'train_lmdb'), db_type='lmdb', device_opts=device_opts)
${tc.join(tc.architectureOutputs, ",", "","")} = self.create_model(train_model, data, device_opts=device_opts, is_test=False)
self.add_training_operators(train_model, ${tc.join(tc.architectureOutputs, ",", "","")}, label, device_opts, loss, opt_type, base_learning_rate, policy, stepsize, epsilon, beta1, beta2, gamma, momentum)
self.add_accuracy(train_model, ${tc.join(tc.architectureOutputs, ",", "","")}, label, device_opts, eval_metric)
if not loss == 'euclidean':
self.add_accuracy(train_model, ${tc.join(tc.architectureOutputs, ",", "","")}, label, device_opts, eval_metric)
with core.DeviceScope(device_opts):
brew.add_weight_decay(train_model, weight_decay)
......@@ -135,38 +146,62 @@ ${tc.include(tc.architecture.body)}
# Main Training Loop
iterations = self.get_total_num_iter(num_epoch, batch_size, train_dataset_size)
print("** Starting Training for " + str(num_epoch) + " epochs = " + str(iterations) + " iterations **")
epoch_as_iter = self.get_epoch_as_iter(num_epoch, batch_size, train_dataset_size)
print("\n*** Starting Training for " + str(num_epoch) + " epochs = " + str(iterations) + " iterations ***")
start_date = datetime.datetime.now()
for i in range(iterations):
workspace.RunNet(train_model.net)
if i % 50 == 0:
print 'Iter ' + str(i) + ': ' + 'Loss ' + str(workspace.FetchBlob("loss")) + ' - ' + 'Accuracy ' + str(workspace.FetchBlob('accuracy'))
print("Training done")
print("== Running Test model ==")
if i % 50 == 0 or i % epoch_as_iter == 0:
if not loss == 'euclidean':
print 'Iter ' + str(i) + ': ' + 'Loss ' + str(workspace.FetchBlob("loss")) + ' - ' + 'Accuracy ' + str(workspace.FetchBlob('accuracy'))
else:
print 'Iter ' + str(i) + ': ' + 'Loss ' + str(workspace.FetchBlob("loss"))
current_time = datetime.datetime.now()
elapsed_time = current_time - start_date
print 'Progress: ' + str(i) + '/' + str(iterations) + ', ' +'Current time spent: ' + str(elapsed_time)
current_time = datetime.datetime.now()
elapsed_time = current_time - start_date
print 'Progress: ' + str(iterations) + '/' + str(iterations) + ' Training done' + ', ' + 'Total time spent: ' + str(elapsed_time)
print("\n*** Running Test model ***")
# == Testing model. ==
test_model= model_helper.ModelHelper(name="test_net", arg_scope=arg_scope, init_params=False)
data, label, test_dataset_size = self.add_input(test_model, batch_size=batch_size, db=os.path.join(self._data_dir_, 'test_lmdb'), db_type='lmdb', device_opts=device_opts)
${tc.join(tc.architectureOutputs, ",", "","")} = self.create_model(test_model, data, device_opts=device_opts, is_test=True)
self.add_accuracy(test_model, predictions, label, device_opts, eval_metric)
if not loss == 'euclidean':
self.add_accuracy(test_model, ${tc.join(tc.architectureOutputs, ",", "","")}, label, device_opts, eval_metric)
workspace.RunNetOnce(test_model.param_init_net)
workspace.CreateNet(test_model.net, overwrite=True)
# Main Testing Loop
test_accuracy = np.zeros(test_dataset_size/batch_size)
start_date = datetime.datetime.now()
for i in range(test_dataset_size/batch_size):
# Run a forward pass of the net on the current batch
workspace.RunNet(test_model.net)
# Collect the batch accuracy from the workspace
test_accuracy[i] = workspace.FetchBlob('accuracy')
print('Test_accuracy: {:.4f}'.format(test_accuracy.mean()))
if not loss == 'euclidean':
test_accuracy[i] = workspace.FetchBlob('accuracy')
print 'Iter ' + str(i) + ': ' + 'Accuracy ' + str(workspace.FetchBlob("accuracy"))
else:
test_accuracy[i] = workspace.FetchBlob("loss")
print 'Iter ' + str(i) + ': ' + 'Loss ' + str(workspace.FetchBlob("loss"))
current_time = datetime.datetime.now()
elapsed_time = current_time - start_date
print 'Progress: ' + str(i) + '/' + str(test_dataset_size/batch_size) + ', ' +'Current time spent: ' + str(elapsed_time)
current_time = datetime.datetime.now()
elapsed_time = current_time - start_date
print 'Progress: ' + str(test_dataset_size/batch_size) + '/' + str(test_dataset_size/batch_size) + ' Testing done' + ', ' + 'Total time spent: ' + str(elapsed_time)
print('Test accuracy mean: {:.9f}'.format(test_accuracy.mean()))
# == Deployment model. ==
# We simply need the main AddModel part.
deploy_model = model_helper.ModelHelper(name="deploy_net", arg_scope=arg_scope, init_params=False)
self.create_model(deploy_model, "data", device_opts, is_test=True)
print("Saving deploy model")
print("\n*** Saving deploy model ***")
self.save_net(self._init_net_, self._predict_net_, deploy_model)
def save_net(self, init_net_path, predict_net_path, model):
......@@ -189,7 +224,7 @@ ${tc.include(tc.architecture.body)}
with open(init_net_path, 'wb') as f:
f.write(init_net.SerializeToString())
print("Save the model to init_net.pbtxt and predict_net.pbtxt")
print("Save the model to init_net.pbtxt and predict_net.pbtxt as additional information")
with open(init_net_path.replace('.pb','.pbtxt'), 'w') as f:
f.write(str(init_net))
......@@ -198,7 +233,6 @@ ${tc.include(tc.architecture.body)}
print("== Saved init_net and predict_net ==")
def load_net(self, init_net_path, predict_net_path, device_opts):
<#--#TODO: Verify that paths ends in '.pb' and not in '.pbtxt'. The extension '.pbtxt' is not supported at the moment.-->
if not os.path.isfile(init_net_path):
logging.error("Network loading failure. File '" + os.path.abspath(init_net_path) + "' does not exist.")
sys.exit(1)
......@@ -217,4 +251,4 @@ ${tc.include(tc.architecture.body)}
net_def.ParseFromString(f.read())
net_def.device_option.CopyFrom(device_opts)
workspace.CreateNet(net_def.SerializeToString(), overwrite=True)
print("== Loaded init_net and predict_net ==")
print("*** Loaded init_net and predict_net ***")
......@@ -3,6 +3,7 @@ from caffe2.python.predictor import mobile_exporter
from caffe2.proto import caffe2_pb2
import numpy as np
import math
import datetime
import logging
import os
import sys
......@@ -28,6 +29,15 @@ class CNNCreator_LeNet:
return iterations_int
def get_epoch_as_iter(self, num_epoch, batch_size, dataset_size): #To print metric durint training process
#Force floating point calculation
batch_size_float = float(batch_size)
dataset_size_float = float(dataset_size)
epoch_float = math.ceil(dataset_size_float/batch_size_float)
epoch_int = int(epoch_float)
return epoch_int
def add_input(self, model, batch_size, db, db_type, device_opts):
with core.DeviceScope(device_opts):
......@@ -142,7 +152,8 @@ class CNNCreator_LeNet:
data, label, train_dataset_size = self.add_input(train_model, batch_size=batch_size, db=os.path.join(self._data_dir_, 'train_lmdb'), db_type='lmdb', device_opts=device_opts)
predictions = self.create_model(train_model, data, device_opts=device_opts, is_test=False)
self.add_training_operators(train_model, predictions, label, device_opts, loss, opt_type, base_learning_rate, policy, stepsize, epsilon, beta1, beta2, gamma, momentum)
self.add_accuracy(train_model, predictions, label, device_opts, eval_metric)
if not loss == 'euclidean':
self.add_accuracy(train_model, predictions, label, device_opts, eval_metric)
with core.DeviceScope(device_opts):
brew.add_weight_decay(train_model, weight_decay)
......@@ -152,38 +163,62 @@ class CNNCreator_LeNet:
# Main Training Loop
iterations = self.get_total_num_iter(num_epoch, batch_size, train_dataset_size)
print("** Starting Training for " + str(num_epoch) + " epochs = " + str(iterations) + " iterations **")
epoch_as_iter = self.get_epoch_as_iter(num_epoch, batch_size, train_dataset_size)
print("\n*** Starting Training for " + str(num_epoch) + " epochs = " + str(iterations) + " iterations ***")
start_date = datetime.datetime.now()
for i in range(iterations):
workspace.RunNet(train_model.net)
if i % 50 == 0:
print 'Iter ' + str(i) + ': ' + 'Loss ' + str(workspace.FetchBlob("loss")) + ' - ' + 'Accuracy ' + str(workspace.FetchBlob('accuracy'))
print("Training done")
print("== Running Test model ==")
if i % 50 == 0 or i % epoch_as_iter == 0:
if not loss == 'euclidean':
print 'Iter ' + str(i) + ': ' + 'Loss ' + str(workspace.FetchBlob("loss")) + ' - ' + 'Accuracy ' + str(workspace.FetchBlob('accuracy'))
else:
print 'Iter ' + str(i) + ': ' + 'Loss ' + str(workspace.FetchBlob("loss"))
current_time = datetime.datetime.now()
elapsed_time = current_time - start_date
print 'Progress: ' + str(i) + '/' + str(iterations) + ', ' +'Current time spent: ' + str(elapsed_time)
current_time = datetime.datetime.now()
elapsed_time = current_time - start_date
print 'Progress: ' + str(iterations) + '/' + str(iterations) + ' Training done' + ', ' + 'Total time spent: ' + str(elapsed_time)
print("\n*** Running Test model ***")
# == Testing model. ==
test_model= model_helper.ModelHelper(name="test_net", arg_scope=arg_scope, init_params=False)
data, label, test_dataset_size = self.add_input(test_model, batch_size=batch_size, db=os.path.join(self._data_dir_, 'test_lmdb'), db_type='lmdb', device_opts=device_opts)
predictions = self.create_model(test_model, data, device_opts=device_opts, is_test=True)
self.add_accuracy(test_model, predictions, label, device_opts, eval_metric)
if not loss == 'euclidean':
self.add_accuracy(test_model, predictions, label, device_opts, eval_metric)
workspace.RunNetOnce(test_model.param_init_net)
workspace.CreateNet(test_model.net, overwrite=True)
# Main Testing Loop
test_accuracy = np.zeros(test_dataset_size/batch_size)
start_date = datetime.datetime.now()
for i in range(test_dataset_size/batch_size):
# Run a forward pass of the net on the current batch
workspace.RunNet(test_model.net)
# Collect the batch accuracy from the workspace
test_accuracy[i] = workspace.FetchBlob('accuracy')
print('Test_accuracy: {:.4f}'.format(test_accuracy.mean()))
if not loss == 'euclidean':
test_accuracy[i] = workspace.FetchBlob('accuracy')
print 'Iter ' + str(i) + ': ' + 'Accuracy ' + str(workspace.FetchBlob("accuracy"))
else:
test_accuracy[i] = workspace.FetchBlob("loss")
print 'Iter ' + str(i) + ': ' + 'Loss ' + str(workspace.FetchBlob("loss"))
current_time = datetime.datetime.now()
elapsed_time = current_time - start_date
print 'Progress: ' + str(i) + '/' + str(test_dataset_size/batch_size) + ', ' +'Current time spent: ' + str(elapsed_time)
current_time = datetime.datetime.now()
elapsed_time = current_time - start_date
print 'Progress: ' + str(test_dataset_size/batch_size) + '/' + str(test_dataset_size/batch_size) + ' Testing done' + ', ' + 'Total time spent: ' + str(elapsed_time)
print('Test accuracy mean: {:.9f}'.format(test_accuracy.mean()))
# == Deployment model. ==
# We simply need the main AddModel part.
deploy_model = model_helper.ModelHelper(name="deploy_net", arg_scope=arg_scope, init_params=False)
self.create_model(deploy_model, "data", device_opts, is_test=True)
print("Saving deploy model")
print("\n*** Saving deploy model ***")
self.save_net(self._init_net_, self._predict_net_, deploy_model)
def save_net(self, init_net_path, predict_net_path, model):
......@@ -206,7 +241,7 @@ class CNNCreator_LeNet:
with open(init_net_path, 'wb') as f:
f.write(init_net.SerializeToString())
print("Save the model to init_net.pbtxt and predict_net.pbtxt")
print("Save the model to init_net.pbtxt and predict_net.pbtxt as additional information")
with open(init_net_path.replace('.pb','.pbtxt'), 'w') as f:
f.write(str(init_net))
......@@ -233,4 +268,4 @@ class CNNCreator_LeNet:
net_def.ParseFromString(f.read())
net_def.device_option.CopyFrom(device_opts)
workspace.CreateNet(net_def.SerializeToString(), overwrite=True)
print("== Loaded init_net and predict_net ==")
print("*** Loaded init_net and predict_net ***")
......@@ -3,6 +3,7 @@ from caffe2.python.predictor import mobile_exporter
from caffe2.proto import caffe2_pb2
import numpy as np
import math
import datetime
import logging
import os
import sys
......@@ -28,6 +29,15 @@ class CNNCreator_VGG16:
return iterations_int
def get_epoch_as_iter(self, num_epoch, batch_size, dataset_size): #To print metric durint training process
#Force floating point calculation
batch_size_float = float(batch_size)
dataset_size_float = float(dataset_size)
epoch_float = math.ceil(dataset_size_float/batch_size_float)
epoch_int = int(epoch_float)
return epoch_int
def add_input(self, model, batch_size, db, db_type, device_opts):
with core.DeviceScope(device_opts):
......@@ -70,7 +80,7 @@ class CNNCreator_VGG16:
conv2_ = brew.conv(model, relu1_, 'conv2_', dim_in=64, dim_out=64, kernel=3, stride=1, pad=1)
# conv2_, output shape: {[64,224,224]}
relu2_ = brew.relu(model, conv2_, conv2_)
pool2_ = brew.max_pool(model, relu2_, 'pool2_', kernel=2, stride=2)
pool2_ = brew.max_pool(model, relu2_, 'pool2_', kernel=2, stride=2, pad=1)
# pool2_, output shape: {[64,112,112]}
conv3_ = brew.conv(model, pool2_, 'conv3_', dim_in=64, dim_out=128, kernel=3, stride=1, pad=1)
# conv3_, output shape: {[128,112,112]}
......@@ -78,7 +88,7 @@ class CNNCreator_VGG16:
conv4_ = brew.conv(model, relu3_, 'conv4_', dim_in=128, dim_out=128, kernel=3, stride=1, pad=1)
# conv4_, output shape: {[128,112,112]}
relu4_ = brew.relu(model, conv4_, conv4_)
pool4_ = brew.max_pool(model, relu4_, 'pool4_', kernel=2, stride=2)
pool4_ = brew.max_pool(model, relu4_, 'pool4_', kernel=2, stride=2, pad=1)
# pool4_, output shape: {[128,56,56]}
conv5_ = brew.conv(model, pool4_, 'conv5_', dim_in=128, dim_out=256, kernel=3, stride=1, pad=1)
# conv5_, output shape: {[256,56,56]}
......@@ -89,7 +99,7 @@ class CNNCreator_VGG16:
conv7_ = brew.conv(model, relu6_, 'conv7_', dim_in=256, dim_out=256, kernel=3, stride=1, pad=1)
# conv7_, output shape: {[256,56,56]}
relu7_ = brew.relu(model, conv7_, conv7_)
pool7_ = brew.max_pool(model, relu7_, 'pool7_', kernel=2, stride=2)
pool7_ = brew.max_pool(model, relu7_, 'pool7_', kernel=2, stride=2, pad=1)
# pool7_, output shape: {[256,28,28]}
conv8_ = brew.conv(model, pool7_, 'conv8_', dim_in=256, dim_out=512, kernel=3, stride=1, pad=1)
# conv8_, output shape: {[512,28,28]}
......@@ -100,7 +110,7 @@ class CNNCreator_VGG16:
conv10_ = brew.conv(model, relu9_, 'conv10_', dim_in=512, dim_out=512, kernel=3, stride=1, pad=1)
# conv10_, output shape: {[512,28,28]}
relu10_ = brew.relu(model, conv10_, conv10_)
pool10_ = brew.max_pool(model, relu10_, 'pool10_', kernel=2, stride=2)
pool10_ = brew.max_pool(model, relu10_, 'pool10_', kernel=2, stride=2, pad=1)
# pool10_, output shape: {[512,14,14]}
conv11_ = brew.conv(model, pool10_, 'conv11_', dim_in=512, dim_out=512, kernel=3, stride=1, pad=1)
# conv11_, output shape: {[512,14,14]}
......@@ -111,7 +121,7 @@ class CNNCreator_VGG16:
conv13_ = brew.conv(model, relu12_, 'conv13_', dim_in=512, dim_out=512, kernel=3, stride=1, pad=1)
# conv13_, output shape: {[512,14,14]}
relu13_ = brew.relu(model, conv13_, conv13_)
pool13_ = brew.max_pool(model, relu13_, 'pool13_', kernel=2, stride=2)
pool13_ = brew.max_pool(model, relu13_, 'pool13_', kernel=2, stride=2, pad=1)
# pool13_, output shape: {[512,7,7]}
fc13_ = brew.fc(model, pool13_, 'fc13_', dim_in=512 * 7 * 7, dim_out=4096)
# fc13_, output shape: {[4096,1,1]}
......@@ -188,7 +198,8 @@ class CNNCreator_VGG16:
data, label, train_dataset_size = self.add_input(train_model, batch_size=batch_size, db=os.path.join(self._data_dir_, 'train_lmdb'), db_type='lmdb', device_opts=device_opts)
predictions = self.create_model(train_model, data, device_opts=device_opts, is_test=False)
self.add_training_operators(train_model, predictions, label, device_opts, loss, opt_type, base_learning_rate, policy, stepsize, epsilon, beta1, beta2, gamma, momentum)
self.add_accuracy(train_model, predictions, label, device_opts, eval_metric)
if not loss == 'euclidean':
self.add_accuracy(train_model, predictions, label, device_opts, eval_metric)
with core.DeviceScope(device_opts):
brew.add_weight_decay(train_model, weight_decay)
......@@ -198,38 +209,62 @@ class CNNCreator_VGG16:
# Main Training Loop
iterations = self.get_total_num_iter(num_epoch, batch_size, train_dataset_size)
print("** Starting Training for " + str(num_epoch) + " epochs = " + str(iterations) + " iterations **")
epoch_as_iter = self.get_epoch_as_iter(num_epoch, batch_size, train_dataset_size)
print("\n*** Starting Training for " + str(num_epoch) + " epochs = " + str(iterations) + " iterations ***")
start_date = datetime.datetime.now()
for i in range(iterations):
workspace.RunNet(train_model.net)
if i % 50 == 0:
print 'Iter ' + str(i) + ': ' + 'Loss ' + str(workspace.FetchBlob("loss")) + ' - ' + 'Accuracy ' + str(workspace.FetchBlob('accuracy'))
print("Training done")
print("== Running Test model ==")
if i % 50 == 0 or i % epoch_as_iter == 0:
if not loss == 'euclidean':
print 'Iter ' + str(i) + ': ' + 'Loss ' + str(workspace.FetchBlob("loss")) + ' - ' + 'Accuracy ' + str(workspace.FetchBlob('accuracy'))
else:
print 'Iter ' + str(i) + ': ' + 'Loss ' + str(workspace.FetchBlob("loss"))
current_time = datetime.datetime.now()
elapsed_time = current_time - start_date
print 'Progress: ' + str(i) + '/' + str(iterations) + ', ' +'Current time spent: ' + str(elapsed_time)
current_time = datetime.datetime.now()
elapsed_time = current_time - start_date
print 'Progress: ' + str(iterations) + '/' + str(iterations) + ' Training done' + ', ' + 'Total time spent: ' + str(elapsed_time)
print("\n*** Running Test model ***")
# == Testing model. ==
test_model= model_helper.ModelHelper(name="test_net", arg_scope=arg_scope, init_params=False)
data, label, test_dataset_size = self.add_input(test_model, batch_size=batch_size, db=os.path.join(self._data_dir_, 'test_lmdb'), db_type='lmdb', device_opts=device_opts)
predictions = self.create_model(test_model, data, device_opts=device_opts, is_test=True)
self.add_accuracy(test_model, predictions, label, device_opts, eval_metric)
if not loss == 'euclidean':
self.add_accuracy(test_model, predictions, label, device_opts, eval_metric)
workspace.RunNetOnce(test_model.param_init_net)
workspace.CreateNet(test_model.net, overwrite=True)
# Main Testing Loop
test_accuracy = np.zeros(test_dataset_size/batch_size)
start_date = datetime.datetime.now()
for i in range(test_dataset_size/batch_size):
# Run a forward pass of the net on the current batch
workspace.RunNet(test_model.net)
# Collect the batch accuracy from the workspace
test_accuracy[i] = workspace.FetchBlob('accuracy')
print('Test_accuracy: {:.4f}'.format(test_accuracy.mean()))
if not loss == 'euclidean':
test_accuracy[i] = workspace.FetchBlob('accuracy')
print 'Iter ' + str(i) + ': ' + 'Accuracy ' + str(workspace.FetchBlob("accuracy"))
else:
test_accuracy[i] = workspace.FetchBlob("loss")
print 'Iter ' + str(i) + ': ' + 'Loss ' + str(workspace.FetchBlob("loss"))
current_time = datetime.datetime.now()
elapsed_time = current_time - start_date
print 'Progress: ' + str(i) + '/' + str(test_dataset_size/batch_size) + ', ' +'Current time spent: ' + str(elapsed_time)
current_time = datetime.datetime.now()
elapsed_time = current_time - start_date
print 'Progress: ' + str(test_dataset_size/batch_size) + '/' + str(test_dataset_size/batch_size) + ' Testing done' + ', ' + 'Total time spent: ' + str(elapsed_time)
print('Test accuracy mean: {:.9f}'.format(test_accuracy.mean()))
# == Deployment model. ==
# We simply need the main AddModel part.
deploy_model = model_helper.ModelHelper(name="deploy_net", arg_scope=arg_scope, init_params=False)
self.create_model(deploy_model, "data", device_opts, is_test=True)
print("Saving deploy model")
print("\n*** Saving deploy model ***")
self.save_net(self._init_net_, self._predict_net_, deploy_model)
def save_net(self, init_net_path, predict_net_path, model):
......@@ -252,7 +287,7 @@ class CNNCreator_VGG16:
with open(init_net_path, 'wb') as f:
f.write(init_net.SerializeToString())
print("Save the model to init_net.pbtxt and predict_net.pbtxt")
print("Save the model to init_net.pbtxt and predict_net.pbtxt as additional information")
with open(init_net_path.replace('.pb','.pbtxt'), 'w') as f:
f.write(str(init_net))
......@@ -279,4 +314,4 @@ class CNNCreator_VGG16:
net_def.ParseFromString(f.read())
net_def.device_option.CopyFrom(device_opts)
workspace.CreateNet(net_def.SerializeToString(), overwrite=True)
print("== Loaded init_net and predict_net ==")
\ No newline at end of file
print("*** Loaded init_net and predict_net ***")
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment