Commit 4be3befa authored by Sebastian N.'s avatar Sebastian N.
Browse files

Merge

parents 35854e1b bdcafb9f
Pipeline #203285 failed with stages
in 25 seconds
package de.monticore.lang.monticar.cnnarch.gluongenerator;
import java.util.HashSet;
import java.util.Set;
public class AllAttentionModels {
public static Set<String> getAttentionModels() {
//List of all models that use attention and should save images of the attention over time
Set models = new HashSet();
models.add("showAttendTell.Show_attend_tell");
return models;
}
}
\ No newline at end of file
......@@ -193,6 +193,7 @@ public class CNNArch2GluonTemplateController extends CNNArchTemplateController {
outputNames.addAll(getStreamLayerVariableMembers(stream, true).keySet());
return outputNames;
}
......@@ -226,8 +227,8 @@ public class CNNArch2GluonTemplateController extends CNNArchTemplateController {
public Map<String, List<String>> getLayerVariableMembers() {
Map<String, List<String>> members = new LinkedHashMap<>();
for (SerialCompositeElementSymbol stream : getArchitecture().getStreams()) {
members.putAll(getStreamLayerVariableMembers(stream, true));
for (NetworkInstructionSymbol networkInstruction : getArchitecture().getNetworkInstructions()) {
members.putAll(getStreamLayerVariableMembers(networkInstruction.getBody(), true));
}
return members;
......@@ -392,6 +393,10 @@ public class CNNArch2GluonTemplateController extends CNNArchTemplateController {
return false;
}
public boolean isAttentionNetwork(){
return AllAttentionModels.getAttentionModels().contains(getComponentName());
}
public int getBeamSearchMaxLength(UnrollInstructionSymbol unroll){
return unroll.getIntValue(AllPredefinedLayers.MAX_LENGTH_NAME).get();
}
......
......@@ -18,12 +18,16 @@ class ${tc.fileNameWithoutEnding}:
train_data = {}
data_mean = {}
data_std = {}
train_images = {}
for input_name in self._input_names_:
train_data[input_name] = train_h5[input_name]
data_mean[input_name + '_'] = nd.array(train_h5[input_name][:].mean(axis=0))
data_std[input_name + '_'] = nd.array(train_h5[input_name][:].std(axis=0) + 1e-5)
if 'images' in train_h5:
train_images = train_h5['images']
train_label = {}
index = 0
for output_name in self._output_names_:
......@@ -42,9 +46,13 @@ class ${tc.fileNameWithoutEnding}:
if test_h5 != None:
test_data = {}
test_images = {}
for input_name in self._input_names_:
test_data[input_name] = test_h5[input_name]
if 'images' in test_h5:
test_images = test_h5['images']
test_label = {}
index = 0
for output_name in self._output_names_:
......@@ -55,7 +63,7 @@ class ${tc.fileNameWithoutEnding}:
label=test_label,
batch_size=test_batch_size)
return train_iter, train_test_iter, test_iter, data_mean, data_std
return train_iter, train_test_iter, test_iter, data_mean, data_std, train_images, test_images
def load_h5_files(self):
train_h5 = None
......
......@@ -96,6 +96,10 @@ ${tc.include(networkInstruction.body, "ARCHITECTURE_DEFINITION")}
def hybrid_forward(self, F, ${tc.join(tc.getStreamInputNames(networkInstruction.body, false), ", ")}):
${tc.include(networkInstruction.body, "FORWARD_FUNCTION")}
<#if tc.isAttentionNetwork() && networkInstruction.isUnroll() >
return ${tc.join(tc.getStreamOutputNames(networkInstruction.body, false), ", ")}, attention_output_
<#else>
return ${tc.join(tc.getStreamOutputNames(networkInstruction.body, false), ", ")}
</#if>
</#list>
......@@ -187,6 +187,7 @@ class ${tc.fileNameWithoutEnding}:
load_checkpoint=True,
context='gpu',
checkpoint_period=5,
save_attention_image=False,
normalize=True):
if context == 'gpu':
mx_context = mx.gpu()
......@@ -213,7 +214,7 @@ class ${tc.fileNameWithoutEnding}:
train_batch_size = batch_size
test_batch_size = ${tc.hasUnrollInstructions()?then('1', 'batch_size')}
train_iter, train_test_iter, test_iter, data_mean, data_std = self._data_loader.load_data(train_batch_size, test_batch_size)
train_iter, train_test_iter, test_iter, data_mean, data_std, train_images, test_images = self._data_loader.load_data(train_batch_size, test_batch_size)
if normalize:
self._net_creator.construct(context=mx_context, data_mean=data_mean, data_std=data_std)
......@@ -239,9 +240,13 @@ class ${tc.fileNameWithoutEnding}:
margin = loss_params['margin'] if 'margin' in loss_params else 1.0
sparseLabel = loss_params['sparse_label'] if 'sparse_label' in loss_params else True
#if loss == 'softmax_cross_entropy':
# fromLogits = loss_params['from_logits'] if 'from_logits' in loss_params else False
# loss_function = mx.gluon.loss.SoftmaxCrossEntropyLoss(from_logits=fromLogits, sparse_label=sparseLabel)
if loss == 'softmax_cross_entropy':
fromLogits = loss_params['from_logits'] if 'from_logits' in loss_params else False
loss_function = mx.gluon.loss.SoftmaxCrossEntropyLoss(from_logits=fromLogits, sparse_label=sparseLabel)
ignore_indices = [2]
loss_function = SoftmaxCrossEntropyLossIgnoreIndices(ignore_indices=ignore_indices, from_logits=fromLogits, sparse_label=sparseLabel)
elif loss == 'sigmoid_binary_cross_entropy':
loss_function = mx.gluon.loss.SigmoidBinaryCrossEntropyLoss()
elif loss == 'cross_entropy':
......@@ -307,6 +312,10 @@ class ${tc.fileNameWithoutEnding}:
if True: <#-- Fix indentation -->
<#include "pythonExecuteTest.ftl">
<#include "saveAttentionImageTrain.ftl">
predictions = []
for output_name in outputs:
if mx.nd.shape_array(mx.nd.squeeze(output_name)).size > 1:
......@@ -323,6 +332,10 @@ class ${tc.fileNameWithoutEnding}:
if True: <#-- Fix indentation -->
<#include "pythonExecuteTest.ftl">
<#include "saveAttentionImageTest.ftl">
predictions = []
for output_name in outputs:
if mx.nd.shape_array(mx.nd.squeeze(output_name)).size > 1:
......
......@@ -36,6 +36,9 @@ if __name__ == "__main__":
<#if (config.normalize)??>
normalize=${config.normalize?string("True","False")},
</#if>
<#if (config.saveAttentionImage)??>
save_attention_image='${config.saveAttentionImage?string("True","False")}',
</#if>
<#if (config.evalMetric)??>
eval_metric='${config.evalMetric.name}',
eval_metric_params={
......
<#-- This template is not used if the following architecture element is an output. See Output.ftl -->
<#if element.axis == -1>
<#assign axis = element.axis?c>
<#else>
<#assign axis = (element.axis + 1)?c>
</#if>
<#assign input = element.inputs[0]>
<#if mode == "FORWARD_FUNCTION">
${element.name} = F.softmax(${input}, axis=${axis})
......
......@@ -22,7 +22,7 @@
</#list>
outputs = []
attentionList=[]
<#list tc.architecture.networkInstructions as networkInstruction>
<#if networkInstruction.isUnroll()>
k = ${tc.getBeamSearchWidth(networkInstruction)}
......@@ -41,7 +41,12 @@
${inputName} = seq[-1]
</#if>
</#list>
<#if tc.isAttentionNetwork()>
${tc.join(tc.getUnrollOutputNames(networkInstruction, "i"), ", ")}, attention_ = self._networks[${networkInstruction?index}](${tc.join(tc.getUnrollInputNames(networkInstruction, "i"), ", ")})
attentionList.append(attention_)
<#else>
${tc.join(tc.getUnrollOutputNames(networkInstruction, "i"), ", ")} = self._networks[${networkInstruction?index}](${tc.join(tc.getUnrollInputNames(networkInstruction, "i"), ", ")})
</#if>
<#list tc.getUnrollOutputNames(networkInstruction, "i") as outputName>
<#if tc.getNameWithoutIndex(outputName) == tc.outputName>
out = ${outputName}
......
......@@ -26,7 +26,11 @@
<#list tc.architecture.networkInstructions as networkInstruction>
<#if networkInstruction.isUnroll()>
for i in range(1, ${tc.getBeamSearchMaxLength(networkInstruction)}):
<#if tc.isAttentionNetwork()>
${tc.join(tc.getUnrollOutputNames(networkInstruction, "i"), ", ")}, _ = self._networks[${networkInstruction?index}](${tc.join(tc.getUnrollInputNames(networkInstruction, "i"), ", ")})
<#else>
${tc.join(tc.getUnrollOutputNames(networkInstruction, "i"), ", ")} = self._networks[${networkInstruction?index}](${tc.join(tc.getUnrollInputNames(networkInstruction, "i"), ", ")})
</#if>
<#list tc.getUnrollOutputNames(networkInstruction, "i") as outputName>
<#if tc.getNameWithoutIndex(outputName) == tc.outputName>
......
if save_attention_image == "True":
plt.clf()
fig = plt.figure(figsize=(10,10))
max_length = len(labels)-1
for l in range(max_length):
attention = attentionList[l]
attention = mx.nd.slice_axis(attention, axis=2, begin=0, end=1)
attention = mx.nd.slice_axis(attention, axis=0, begin=0, end=1)
attention = mx.nd.squeeze(attention)
attention_resized = np.resize(attention.asnumpy(), (8, 8))
ax = fig.add_subplot(max_length//3, max_length//4, l+1)
ax.set_title(dict[int(mx.nd.slice_axis(mx.nd.argmax(outputs[l+1], axis=1), axis=0, begin=0, end=1).asscalar())])
img = ax.imshow(test_images[0+batch_size*(batch_i)])
ax.imshow(attention_resized, cmap='gray', alpha=0.6, extent=img.get_extent())
plt.tight_layout()
plt.savefig(target_dir + '/attention_test.png')
plt.close()
\ No newline at end of file
if save_attention_image == "True":
import matplotlib.pyplot as plt
logging.getLogger('matplotlib').setLevel(logging.ERROR)
plt.clf()
fig = plt.figure(figsize=(10,10))
max_length = len(labels)-1
if(os.path.isfile('src/test/resources/training_data/Show_attend_tell/dict.pkl')):
with open('src/test/resources/training_data/Show_attend_tell/dict.pkl', 'rb') as f:
dict = pickle.load(f)
for l in range(max_length):
attention = attentionList[l]
attention = mx.nd.slice_axis(attention, axis=0, begin=0, end=1)
attention = mx.nd.squeeze(attention)
attention_resized = np.resize(attention.asnumpy(), (8, 8))
ax = fig.add_subplot(max_length//3, max_length//4, l+1)
ax.set_title(dict[int(labels[l+1][0].asscalar())])
img = ax.imshow(train_images[0+batch_size*(batch_i)])
ax.imshow(attention_resized, cmap='gray', alpha=0.6, extent=img.get_extent())
plt.tight_layout()
target_dir = 'target/attention_images'
if not os.path.exists(target_dir):
os.makedirs(target_dir)
plt.savefig(target_dir + '/attention_train.png')
plt.close()
\ No newline at end of file
......@@ -18,12 +18,16 @@ class CNNDataLoader_Alexnet:
train_data = {}
data_mean = {}
data_std = {}
train_images = {}
for input_name in self._input_names_:
train_data[input_name] = train_h5[input_name]
data_mean[input_name + '_'] = nd.array(train_h5[input_name][:].mean(axis=0))
data_std[input_name + '_'] = nd.array(train_h5[input_name][:].std(axis=0) + 1e-5)
if 'images' in train_h5:
train_images = train_h5['images']
train_label = {}
index = 0
for output_name in self._output_names_:
......@@ -42,9 +46,13 @@ class CNNDataLoader_Alexnet:
if test_h5 != None:
test_data = {}
test_images = {}
for input_name in self._input_names_:
test_data[input_name] = test_h5[input_name]
if 'images' in test_h5:
test_images = test_h5['images']
test_label = {}
index = 0
for output_name in self._output_names_:
......@@ -55,7 +63,7 @@ class CNNDataLoader_Alexnet:
label=test_label,
batch_size=test_batch_size)
return train_iter, train_test_iter, test_iter, data_mean, data_std
return train_iter, train_test_iter, test_iter, data_mean, data_std, train_images, test_images
def load_h5_files(self):
train_h5 = None
......
......@@ -18,12 +18,16 @@ class CNNDataLoader_CifarClassifierNetwork:
train_data = {}
data_mean = {}
data_std = {}
train_images = {}
for input_name in self._input_names_:
train_data[input_name] = train_h5[input_name]
data_mean[input_name + '_'] = nd.array(train_h5[input_name][:].mean(axis=0))
data_std[input_name + '_'] = nd.array(train_h5[input_name][:].std(axis=0) + 1e-5)
if 'images' in train_h5:
train_images = train_h5['images']
train_label = {}
index = 0
for output_name in self._output_names_:
......@@ -42,9 +46,13 @@ class CNNDataLoader_CifarClassifierNetwork:
if test_h5 != None:
test_data = {}
test_images = {}
for input_name in self._input_names_:
test_data[input_name] = test_h5[input_name]
if 'images' in test_h5:
test_images = test_h5['images']
test_label = {}
index = 0
for output_name in self._output_names_:
......@@ -55,7 +63,7 @@ class CNNDataLoader_CifarClassifierNetwork:
label=test_label,
batch_size=test_batch_size)
return train_iter, train_test_iter, test_iter, data_mean, data_std
return train_iter, train_test_iter, test_iter, data_mean, data_std, train_images, test_images
def load_h5_files(self):
train_h5 = None
......
......@@ -18,12 +18,16 @@ class CNNDataLoader_VGG16:
train_data = {}
data_mean = {}
data_std = {}
train_images = {}
for input_name in self._input_names_:
train_data[input_name] = train_h5[input_name]
data_mean[input_name + '_'] = nd.array(train_h5[input_name][:].mean(axis=0))
data_std[input_name + '_'] = nd.array(train_h5[input_name][:].std(axis=0) + 1e-5)
if 'images' in train_h5:
train_images = train_h5['images']
train_label = {}
index = 0
for output_name in self._output_names_:
......@@ -42,9 +46,13 @@ class CNNDataLoader_VGG16:
if test_h5 != None:
test_data = {}
test_images = {}
for input_name in self._input_names_:
test_data[input_name] = test_h5[input_name]
if 'images' in test_h5:
test_images = test_h5['images']
test_label = {}
index = 0
for output_name in self._output_names_:
......@@ -55,7 +63,7 @@ class CNNDataLoader_VGG16:
label=test_label,
batch_size=test_batch_size)
return train_iter, train_test_iter, test_iter, data_mean, data_std
return train_iter, train_test_iter, test_iter, data_mean, data_std, train_images, test_images
def load_h5_files(self):
train_h5 = None
......
......@@ -187,6 +187,7 @@ class CNNSupervisedTrainer_Alexnet:
load_checkpoint=True,
context='gpu',
checkpoint_period=5,
save_attention_image=False,
normalize=True):
if context == 'gpu':
mx_context = mx.gpu()
......@@ -213,7 +214,7 @@ class CNNSupervisedTrainer_Alexnet:
train_batch_size = batch_size
test_batch_size = batch_size
train_iter, train_test_iter, test_iter, data_mean, data_std = self._data_loader.load_data(train_batch_size, test_batch_size)
train_iter, train_test_iter, test_iter, data_mean, data_std, train_images, test_images = self._data_loader.load_data(train_batch_size, test_batch_size)
if normalize:
self._net_creator.construct(context=mx_context, data_mean=data_mean, data_std=data_std)
......@@ -239,9 +240,13 @@ class CNNSupervisedTrainer_Alexnet:
margin = loss_params['margin'] if 'margin' in loss_params else 1.0
sparseLabel = loss_params['sparse_label'] if 'sparse_label' in loss_params else True
#if loss == 'softmax_cross_entropy':
# fromLogits = loss_params['from_logits'] if 'from_logits' in loss_params else False
# loss_function = mx.gluon.loss.SoftmaxCrossEntropyLoss(from_logits=fromLogits, sparse_label=sparseLabel)
if loss == 'softmax_cross_entropy':
fromLogits = loss_params['from_logits'] if 'from_logits' in loss_params else False
loss_function = mx.gluon.loss.SoftmaxCrossEntropyLoss(from_logits=fromLogits, sparse_label=sparseLabel)
ignore_indices = [2]
loss_function = SoftmaxCrossEntropyLossIgnoreIndices(ignore_indices=ignore_indices, from_logits=fromLogits, sparse_label=sparseLabel)
elif loss == 'sigmoid_binary_cross_entropy':
loss_function = mx.gluon.loss.SigmoidBinaryCrossEntropyLoss()
elif loss == 'cross_entropy':
......@@ -324,11 +329,42 @@ class CNNSupervisedTrainer_Alexnet:
outputs = []
attentionList=[]
predictions_ = self._networks[0](data_)
outputs.append(predictions_)
if save_attention_image == "True":
import matplotlib.pyplot as plt
logging.getLogger('matplotlib').setLevel(logging.ERROR)
plt.clf()
fig = plt.figure(figsize=(10,10))
max_length = len(labels)-1
if(os.path.isfile('src/test/resources/training_data/Show_attend_tell/dict.pkl')):
with open('src/test/resources/training_data/Show_attend_tell/dict.pkl', 'rb') as f:
dict = pickle.load(f)
for l in range(max_length):
attention = attentionList[l]
attention = mx.nd.slice_axis(attention, axis=0, begin=0, end=1)
attention = mx.nd.squeeze(attention)
attention_resized = np.resize(attention.asnumpy(), (8, 8))
ax = fig.add_subplot(max_length//3, max_length//4, l+1)
ax.set_title(dict[int(labels[l+1][0].asscalar())])
img = ax.imshow(train_images[0+batch_size*(batch_i)])
ax.imshow(attention_resized, cmap='gray', alpha=0.6, extent=img.get_extent())
plt.tight_layout()
target_dir = 'target/attention_images'
if not os.path.exists(target_dir):
os.makedirs(target_dir)
plt.savefig(target_dir + '/attention_train.png')
plt.close()
predictions = []
for output_name in outputs:
if mx.nd.shape_array(mx.nd.squeeze(output_name)).size > 1:
......@@ -351,11 +387,33 @@ class CNNSupervisedTrainer_Alexnet:
outputs = []
attentionList=[]
predictions_ = self._networks[0](data_)
outputs.append(predictions_)
if save_attention_image == "True":
plt.clf()
fig = plt.figure(figsize=(10,10))
max_length = len(labels)-1
for l in range(max_length):
attention = attentionList[l]
attention = mx.nd.slice_axis(attention, axis=2, begin=0, end=1)
attention = mx.nd.slice_axis(attention, axis=0, begin=0, end=1)
attention = mx.nd.squeeze(attention)
attention_resized = np.resize(attention.asnumpy(), (8, 8))
ax = fig.add_subplot(max_length//3, max_length//4, l+1)
ax.set_title(dict[int(mx.nd.slice_axis(mx.nd.argmax(outputs[l+1], axis=1), axis=0, begin=0, end=1).asscalar())])
img = ax.imshow(test_images[0+batch_size*(batch_i)])
ax.imshow(attention_resized, cmap='gray', alpha=0.6, extent=img.get_extent())
plt.tight_layout()
plt.savefig(target_dir + '/attention_test.png')
plt.close()
predictions = []
for output_name in outputs:
if mx.nd.shape_array(mx.nd.squeeze(output_name)).size > 1:
......
......@@ -187,6 +187,7 @@ class CNNSupervisedTrainer_CifarClassifierNetwork:
load_checkpoint=True,
context='gpu',
checkpoint_period=5,
save_attention_image=False,
normalize=True):
if context == 'gpu':
mx_context = mx.gpu()
......@@ -213,7 +214,7 @@ class CNNSupervisedTrainer_CifarClassifierNetwork:
train_batch_size = batch_size
test_batch_size = batch_size
train_iter, train_test_iter, test_iter, data_mean, data_std = self._data_loader.load_data(train_batch_size, test_batch_size)
train_iter, train_test_iter, test_iter, data_mean, data_std, train_images, test_images = self._data_loader.load_data(train_batch_size, test_batch_size)
if normalize:
self._net_creator.construct(context=mx_context, data_mean=data_mean, data_std=data_std)
......@@ -239,9 +240,13 @@ class CNNSupervisedTrainer_CifarClassifierNetwork:
margin = loss_params['margin'] if 'margin' in loss_params else 1.0
sparseLabel = loss_params['sparse_label'] if 'sparse_label' in loss_params else True
#if loss == 'softmax_cross_entropy':
# fromLogits = loss_params['from_logits'] if 'from_logits' in loss_params else False
# loss_function = mx.gluon.loss.SoftmaxCrossEntropyLoss(from_logits=fromLogits, sparse_label=sparseLabel)
if loss == 'softmax_cross_entropy':
fromLogits = loss_params['from_logits'] if 'from_logits' in loss_params else False
loss_function = mx.gluon.loss.SoftmaxCrossEntropyLoss(from_logits=fromLogits, sparse_label=sparseLabel)
ignore_indices = [2]
loss_function = SoftmaxCrossEntropyLossIgnoreIndices(ignore_indices=ignore_indices, from_logits=fromLogits, sparse_label=sparseLabel)
elif loss == 'sigmoid_binary_cross_entropy':
loss_function = mx.gluon.loss.SigmoidBinaryCrossEntropyLoss()
elif loss == 'cross_entropy':
......@@ -324,11 +329,42 @@ class CNNSupervisedTrainer_CifarClassifierNetwork:
outputs = []
attentionList=[]
softmax_ = self._networks[0](data_)
outputs.append(softmax_)
if save_attention_image == "True":
import matplotlib.pyplot as plt
logging.getLogger('matplotlib').setLevel(logging.ERROR)
plt.clf()
fig = plt.figure(figsize=(10,10))
max_length = len(labels)-1
if(os.path.isfile('src/test/resources/training_data/Show_attend_tell/dict.pkl')):
with open('src/test/resources/training_data/Show_attend_tell/dict.pkl', 'rb') as f:
dict = pickle.load(f)
for l in range(max_length):
attention = attentionList[l]
attention = mx.nd.slice_axis(attention, axis=0, begin=0, end=1)
attention = mx.nd.squeeze(attention)
attention_resized = np.resize(attention.asnumpy(), (8, 8))
ax = fig.add_subplot(max_length//3, max_length//4, l+1)
ax.set_title(dict[int(labels[l+1][0].asscalar())])
img = ax.imshow(train_images[0+batch_size*(batch_i)])
ax.imshow(attention_resized, cmap='gray', alpha=0.6, extent=img.get_extent())
plt.tight_layout()
target_dir = 'target/attention_images'
if not os.path.exists(target_dir):
os.makedirs(target_dir)
plt.savefig(target_dir + '/attention_train.png')
plt.close()
predictions = []
for output_name in outputs:
if mx.nd.shape_array(mx.nd.squeeze(output_name)).size > 1:
......@@ -351,11 +387,33 @@ class CNNSupervisedTrainer_CifarClassifierNetwork:
outputs = []
attentionList=[]
softmax_ = self._networks[0](data_)
outputs.append(softmax_)
if save_attention_image == "True":
plt.clf()
fig = plt.figure(figsize=(10,10))
max_length = len(labels)-1