Skip to content
GitLab
Commit 58436d17 authored by Sebastian Nickels's avatar Sebastian Nickels
Browse files

Added output of train and test/validation loss at the end of each epoch

parent bf56b53c
Pipeline #226535 failed with stages
in 33 seconds
Hide whitespace changes
Inline Side-by-side
src/main/resources/templates/gluon/CNNSupervisedTrainer.ftl
View file @ 58436d17
......@@ -291,6 +291,9 @@ class ${tc.fileNameWithoutEnding}:
else:
train_iter, test_iter, data_mean, data_std, train_images, test_images = self._data_loader.load_data(batch_size, shuffle_data)
global_loss_train = 0.0
train_batches = 0
loss_total = 0
train_iter.reset()
for batch_i, batch in enumerate(train_iter):
......@@ -305,6 +308,9 @@ class ${tc.fileNameWithoutEnding}:
loss_total += loss.sum().asscalar()
global_loss_train += float(loss.mean().asscalar())
train_batches += 1
if clip_global_grad_norm:
grads = []
......@@ -332,6 +338,9 @@ class ${tc.fileNameWithoutEnding}:
tic = time.time()
if train_batches > 0:
global_loss_train /= train_batches
tic = None
......@@ -357,6 +366,9 @@ class ${tc.fileNameWithoutEnding}:
else:
train_metric_score = 0
global_loss_test = 0.0
test_batches = 0
test_iter.reset()
metric = mx.metric.create(eval_metric, **eval_metric_params)
for batch_i, batch in enumerate(test_iter):
......@@ -366,6 +378,12 @@ class ${tc.fileNameWithoutEnding}:
<#include "saveAttentionImageTest.ftl">
loss = 0
for element in lossList:
loss = loss + element
global_loss_test += float(loss.mean().asscalar())
test_batches += 1
predictions = []
for output_name in outputs:
......@@ -378,8 +396,10 @@ class ${tc.fileNameWithoutEnding}:
metric.update(preds=predictions, labels=labels)
test_metric_score = metric.get()[1]
logging.info("Epoch[%d] Train: %f, Test: %f" % (epoch, train_metric_score, test_metric_score))
if test_batches > 0:
global_loss_test /= test_batches
logging.info("Epoch[%d] Train: %f, Test: %f, Train Loss: %f, Test Loss: %f" % (epoch, train_metric_score, test_metric_score, global_loss_train, global_loss_test))
if (epoch - begin_epoch) % checkpoint_period == 0:
for i, network in self._networks.items():
......
src/main/resources/templates/gluon/pythonExecuteTest.ftl
View file @ 58436d17
......@@ -24,7 +24,8 @@
nd.waitall()
outputs = []
attentionList=[]
lossList = []
attentionList = []
<#list tc.architecture.networkInstructions as networkInstruction>
<#if networkInstruction.isUnroll()>
k = ${tc.getBeamSearchWidth(networkInstruction)}
......@@ -92,6 +93,7 @@
<#if tc.getNameWithoutIndex(outputName) == tc.outputName>
${outputName} = sequences[0][0][i]
outputs.append(${outputName})
lossList.append(loss_function(${outputName}, labels[${tc.getIndex(outputName, true)}]))
<#if tc.isAttentionNetwork()>
attentionList.append(sequences[0][2][i])
</#if>
......@@ -103,6 +105,7 @@
<#list tc.getStreamOutputNames(networkInstruction.body, true) as outputName>
<#if tc.getNameWithoutIndex(outputName) == tc.outputName>
outputs.append(${outputName})
lossList.append(loss_function(${outputName}, labels[${tc.getIndex(outputName, true)}]))
<#if tc.endsWithArgmax(networkInstruction.body)>
${outputName} = mx.nd.argmax(${outputName}, axis=1).expand_dims(1)
</#if>
......
src/test/resources/target_code/CNNSupervisedTrainer_Alexnet.py
View file @ 58436d17
......@@ -283,6 +283,15 @@ class CNNSupervisedTrainer_Alexnet:
tic = None
for epoch in range(begin_epoch, begin_epoch + num_epoch):
if shuffle_data:
if preprocessing:
preproc_lib = "CNNPreprocessor_Alexnet_executor"
train_iter, test_iter, data_mean, data_std, train_images, test_images = self._data_loader.load_preprocessed_data(batch_size, preproc_lib, shuffle_data)
else:
train_iter, test_iter, data_mean, data_std, train_images, test_images = self._data_loader.load_data(batch_size, shuffle_data)
global_loss_train = 0.0
train_batches = 0
loss_total = 0
train_iter.reset()
......@@ -311,6 +320,9 @@ class CNNSupervisedTrainer_Alexnet:
loss_total += loss.sum().asscalar()
global_loss_train += float(loss.mean().asscalar())
train_batches += 1
if clip_global_grad_norm:
grads = []
......@@ -338,6 +350,9 @@ class CNNSupervisedTrainer_Alexnet:
tic = time.time()
if train_batches > 0:
global_loss_train /= train_batches
tic = None
......@@ -355,10 +370,12 @@ class CNNSupervisedTrainer_Alexnet:
nd.waitall()
outputs = []
attentionList=[]
lossList = []
attentionList = []
predictions_ = self._networks[0](data_)
outputs.append(predictions_)
lossList.append(loss_function(predictions_, labels[0]))
if save_attention_image == "True":
......@@ -414,10 +431,13 @@ class CNNSupervisedTrainer_Alexnet:
else:
train_metric_score = 0
global_loss_test = 0.0
test_batches = 0
test_iter.reset()
metric = mx.metric.create(eval_metric, **eval_metric_params)
for batch_i, batch in enumerate(test_iter):
if True:
if True:
labels = [batch.label[i].as_in_context(mx_context) for i in range(1)]
data_ = batch.data[0].as_in_context(mx_context)
......@@ -428,10 +448,12 @@ class CNNSupervisedTrainer_Alexnet:
nd.waitall()
outputs = []
attentionList=[]
lossList = []
attentionList = []
predictions_ = self._networks[0](data_)
outputs.append(predictions_)
lossList.append(loss_function(predictions_, labels[0]))
if save_attention_image == "True":
......@@ -475,6 +497,12 @@ class CNNSupervisedTrainer_Alexnet:
os.makedirs(target_dir)
plt.savefig(target_dir + '/attention_test.png')
plt.close()
loss = 0
for element in lossList:
loss = loss + element
global_loss_test += float(loss.mean().asscalar())
test_batches += 1
predictions = []
for output_name in outputs:
......@@ -487,8 +515,10 @@ class CNNSupervisedTrainer_Alexnet:
metric.update(preds=predictions, labels=labels)
test_metric_score = metric.get()[1]
logging.info("Epoch[%d] Train: %f, Test: %f" % (epoch, train_metric_score, test_metric_score))
if test_batches > 0:
global_loss_test /= test_batches
logging.info("Epoch[%d] Train: %f, Test: %f, Train Loss: %f, Test Loss: %f" % (epoch, train_metric_score, test_metric_score, global_loss_train, global_loss_test))
if (epoch - begin_epoch) % checkpoint_period == 0:
for i, network in self._networks.items():
......
src/test/resources/target_code/CNNSupervisedTrainer_CifarClassifierNetwork.py
View file @ 58436d17
......@@ -283,6 +283,15 @@ class CNNSupervisedTrainer_CifarClassifierNetwork:
tic = None
for epoch in range(begin_epoch, begin_epoch + num_epoch):
if shuffle_data:
if preprocessing:
preproc_lib = "CNNPreprocessor_CifarClassifierNetwork_executor"
train_iter, test_iter, data_mean, data_std, train_images, test_images = self._data_loader.load_preprocessed_data(batch_size, preproc_lib, shuffle_data)
else:
train_iter, test_iter, data_mean, data_std, train_images, test_images = self._data_loader.load_data(batch_size, shuffle_data)
global_loss_train = 0.0
train_batches = 0
loss_total = 0
train_iter.reset()
......@@ -311,6 +320,9 @@ class CNNSupervisedTrainer_CifarClassifierNetwork:
loss_total += loss.sum().asscalar()
global_loss_train += float(loss.mean().asscalar())
train_batches += 1
if clip_global_grad_norm:
grads = []
......@@ -338,6 +350,9 @@ class CNNSupervisedTrainer_CifarClassifierNetwork:
tic = time.time()
if train_batches > 0:
global_loss_train /= train_batches
tic = None
......@@ -355,10 +370,12 @@ class CNNSupervisedTrainer_CifarClassifierNetwork:
nd.waitall()
outputs = []
attentionList=[]
lossList = []
attentionList = []
softmax_ = self._networks[0](data_)
outputs.append(softmax_)
lossList.append(loss_function(softmax_, labels[0]))
if save_attention_image == "True":
......@@ -414,10 +431,13 @@ class CNNSupervisedTrainer_CifarClassifierNetwork:
else:
train_metric_score = 0
global_loss_test = 0.0
test_batches = 0
test_iter.reset()
metric = mx.metric.create(eval_metric, **eval_metric_params)
for batch_i, batch in enumerate(test_iter):
if True:
if True:
labels = [batch.label[i].as_in_context(mx_context) for i in range(1)]
data_ = batch.data[0].as_in_context(mx_context)
......@@ -428,10 +448,12 @@ class CNNSupervisedTrainer_CifarClassifierNetwork:
nd.waitall()
outputs = []
attentionList=[]
lossList = []
attentionList = []
softmax_ = self._networks[0](data_)
outputs.append(softmax_)
lossList.append(loss_function(softmax_, labels[0]))
if save_attention_image == "True":
......@@ -475,6 +497,12 @@ class CNNSupervisedTrainer_CifarClassifierNetwork:
os.makedirs(target_dir)
plt.savefig(target_dir + '/attention_test.png')
plt.close()
loss = 0
for element in lossList:
loss = loss + element
global_loss_test += float(loss.mean().asscalar())
test_batches += 1
predictions = []
for output_name in outputs:
......@@ -487,8 +515,10 @@ class CNNSupervisedTrainer_CifarClassifierNetwork:
metric.update(preds=predictions, labels=labels)
test_metric_score = metric.get()[1]
logging.info("Epoch[%d] Train: %f, Test: %f" % (epoch, train_metric_score, test_metric_score))
if test_batches > 0:
global_loss_test /= test_batches
logging.info("Epoch[%d] Train: %f, Test: %f, Train Loss: %f, Test Loss: %f" % (epoch, train_metric_score, test_metric_score, global_loss_train, global_loss_test))
if (epoch - begin_epoch) % checkpoint_period == 0:
for i, network in self._networks.items():
......
src/test/resources/target_code/CNNSupervisedTrainer_VGG16.py
View file @ 58436d17
......@@ -283,6 +283,15 @@ class CNNSupervisedTrainer_VGG16:
tic = None
for epoch in range(begin_epoch, begin_epoch + num_epoch):
if shuffle_data:
if preprocessing:
preproc_lib = "CNNPreprocessor_VGG16_executor"
train_iter, test_iter, data_mean, data_std, train_images, test_images = self._data_loader.load_preprocessed_data(batch_size, preproc_lib, shuffle_data)
else:
train_iter, test_iter, data_mean, data_std, train_images, test_images = self._data_loader.load_data(batch_size, shuffle_data)
global_loss_train = 0.0
train_batches = 0
loss_total = 0
train_iter.reset()
......@@ -311,6 +320,9 @@ class CNNSupervisedTrainer_VGG16:
loss_total += loss.sum().asscalar()
global_loss_train += float(loss.mean().asscalar())
train_batches += 1
if clip_global_grad_norm:
grads = []
......@@ -338,6 +350,9 @@ class CNNSupervisedTrainer_VGG16:
tic = time.time()
if train_batches > 0:
global_loss_train /= train_batches
tic = None
......@@ -355,10 +370,12 @@ class CNNSupervisedTrainer_VGG16:
nd.waitall()
outputs = []
attentionList=[]
lossList = []
attentionList = []
predictions_ = self._networks[0](data_)
outputs.append(predictions_)
lossList.append(loss_function(predictions_, labels[0]))
if save_attention_image == "True":
......@@ -414,10 +431,13 @@ class CNNSupervisedTrainer_VGG16:
else:
train_metric_score = 0
global_loss_test = 0.0
test_batches = 0
test_iter.reset()
metric = mx.metric.create(eval_metric, **eval_metric_params)
for batch_i, batch in enumerate(test_iter):
if True:
if True:
labels = [batch.label[i].as_in_context(mx_context) for i in range(1)]
data_ = batch.data[0].as_in_context(mx_context)
......@@ -428,10 +448,12 @@ class CNNSupervisedTrainer_VGG16:
nd.waitall()
outputs = []
attentionList=[]
lossList = []
attentionList = []
predictions_ = self._networks[0](data_)
outputs.append(predictions_)
lossList.append(loss_function(predictions_, labels[0]))
if save_attention_image == "True":
......@@ -475,6 +497,12 @@ class CNNSupervisedTrainer_VGG16:
os.makedirs(target_dir)
plt.savefig(target_dir + '/attention_test.png')
plt.close()
loss = 0
for element in lossList:
loss = loss + element
global_loss_test += float(loss.mean().asscalar())
test_batches += 1
predictions = []
for output_name in outputs:
......@@ -487,8 +515,10 @@ class CNNSupervisedTrainer_VGG16:
metric.update(preds=predictions, labels=labels)
test_metric_score = metric.get()[1]
logging.info("Epoch[%d] Train: %f, Test: %f" % (epoch, train_metric_score, test_metric_score))
if test_batches > 0:
global_loss_test /= test_batches
logging.info("Epoch[%d] Train: %f, Test: %f, Train Loss: %f, Test Loss: %f" % (epoch, train_metric_score, test_metric_score, global_loss_train, global_loss_test))
if (epoch - begin_epoch) % checkpoint_period == 0:
for i, network in self._networks.items():
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment