Commit bdcafb9f authored by Christian Fuß's avatar Christian Fuß
Browse files

Added possibility to save the visual attention for Show, attend and tell...

Added possibility to save the visual attention for Show, attend and tell architecture as an image. Adjusted axis parameter in softmax layer to ignore batch size. Fixed a problem with some states not being initialized correctly in C++ code due to previous changes to the inline mode
parent 23631e61
Pipeline #202128 failed with stages
in 23 seconds
package de.monticore.lang.monticar.cnnarch.gluongenerator;
import java.util.HashSet;
import java.util.Set;
public class AllAttentionModels {
public static Set<String> getAttentionModels() {
//List of all models that use attention and should save images of the attention over time
Set models = new HashSet();
models.add("showAttendTell.Show_attend_tell");
return models;
}
}
\ No newline at end of file
......@@ -159,7 +159,8 @@ public class CNNArch2GluonTemplateController extends CNNArchTemplateController {
}
}
outputNames.addAll(getStreamLayerVariableMembers(stream, true).keySet());
outputNames.addAll(getStreamLayerVariableMembers(stream, true, false).keySet());
return outputNames;
}
......@@ -179,11 +180,11 @@ public class CNNArch2GluonTemplateController extends CNNArchTemplateController {
}
// Used to initialize all layer variable members which are passed through the networks
public Map<String, List<String>> getLayerVariableMembers() {
public Map<String, List<String>> getLayerVariableMembers(boolean generateStateInitializers) {
Map<String, List<String>> members = new LinkedHashMap<>();
for (SerialCompositeElementSymbol stream : getArchitecture().getStreams()) {
members.putAll(getStreamLayerVariableMembers(stream, true));
members.putAll(getStreamLayerVariableMembers(stream, true, generateStateInitializers));
}
return members;
......@@ -236,12 +237,12 @@ public class CNNArch2GluonTemplateController extends CNNArchTemplateController {
}
}
inputs.putAll(getStreamLayerVariableMembers(stream, false));
inputs.putAll(getStreamLayerVariableMembers(stream, false, false));
return inputs;
}
private Map<String, List<String>> getStreamLayerVariableMembers(SerialCompositeElementSymbol stream, boolean includeOutput) {
private Map<String, List<String>> getStreamLayerVariableMembers(SerialCompositeElementSymbol stream, boolean includeOutput, boolean generateStateInitializaters) {
Map<String, List<String>> members = new LinkedHashMap<>();
List<ArchitectureElementSymbol> elements = stream.getSpannedScope().resolveLocally(ArchitectureElementSymbol.KIND);
......@@ -249,7 +250,7 @@ public class CNNArch2GluonTemplateController extends CNNArchTemplateController {
if (element instanceof VariableSymbol) {
VariableSymbol variable = (VariableSymbol) element;
if (variable.getType() == VariableSymbol.Type.LAYER && (variable.getMember() == VariableSymbol.Member.NONE)) {
if (variable.getType() == VariableSymbol.Type.LAYER && (variable.getMember() == VariableSymbol.Member.NONE || generateStateInitializaters)) {
LayerVariableDeclarationSymbol layerVariableDeclaration = variable.getLayerVariableDeclaration();
if (layerVariableDeclaration.getLayer().getDeclaration().isPredefined()) {
......@@ -321,6 +322,10 @@ public class CNNArch2GluonTemplateController extends CNNArchTemplateController {
return dimensions;
}
public boolean isAttentionNetwork(){
return AllAttentionModels.getAttentionModels().contains(getComponentName());
}
public int getBeamSearchWidth(UnrollInstructionSymbol unroll){
return unroll.getIntValue(AllPredefinedLayers.WIDTH_NAME).get();
}
......
......@@ -18,12 +18,16 @@ class ${tc.fileNameWithoutEnding}:
train_data = {}
data_mean = {}
data_std = {}
train_images = {}
for input_name in self._input_names_:
train_data[input_name] = train_h5[input_name]
data_mean[input_name + '_'] = nd.array(train_h5[input_name][:].mean(axis=0))
data_std[input_name + '_'] = nd.array(train_h5[input_name][:].std(axis=0) + 1e-5)
if 'images' in train_h5:
train_images = train_h5['images']
train_label = {}
index = 0
for output_name in self._output_names_:
......@@ -38,9 +42,13 @@ class ${tc.fileNameWithoutEnding}:
if test_h5 != None:
test_data = {}
test_images = {}
for input_name in self._input_names_:
test_data[input_name] = test_h5[input_name]
if 'images' in test_h5:
test_images = test_h5['images']
test_label = {}
index = 0
for output_name in self._output_names_:
......@@ -52,7 +60,7 @@ class ${tc.fileNameWithoutEnding}:
label=test_label,
batch_size=batch_size)
return train_iter, test_iter, data_mean, data_std
return train_iter, test_iter, data_mean, data_std, train_images, test_images
def load_h5_files(self):
train_h5 = None
......
......@@ -98,7 +98,11 @@ ${tc.include(networkInstruction.body, "ARCHITECTURE_DEFINITION")}
def hybrid_forward(self, F, ${tc.join(tc.getStreamInputNames(networkInstruction.body), ", ")}):
${tc.include(networkInstruction.body, "FORWARD_FUNCTION")}
<#if tc.isAttentionNetwork() && networkInstruction.isUnroll() >
return ${tc.join(tc.getStreamOutputNames(networkInstruction.body), ", ")}, attention_output_
<#else>
return ${tc.join(tc.getStreamOutputNames(networkInstruction.body), ", ")}
</#if>
</#if>
</#list>
......@@ -220,6 +220,7 @@ class ${tc.fileNameWithoutEnding}:
load_checkpoint=True,
context='gpu',
checkpoint_period=5,
save_attention_image=False,
normalize=True):
if context == 'gpu':
mx_context = mx.gpu()
......@@ -244,7 +245,7 @@ class ${tc.fileNameWithoutEnding}:
del optimizer_params['learning_rate_decay']
train_iter, test_iter, data_mean, data_std = self._data_loader.load_data(batch_size)
train_iter, test_iter, data_mean, data_std, train_images, test_images = self._data_loader.load_data(batch_size)
if normalize:
self._net_creator.construct(context=mx_context, data_mean=data_mean, data_std=data_std)
......@@ -270,9 +271,13 @@ class ${tc.fileNameWithoutEnding}:
margin = loss_params['margin'] if 'margin' in loss_params else 1.0
sparseLabel = loss_params['sparse_label'] if 'sparse_label' in loss_params else True
#if loss == 'softmax_cross_entropy':
# fromLogits = loss_params['from_logits'] if 'from_logits' in loss_params else False
# loss_function = mx.gluon.loss.SoftmaxCrossEntropyLoss(from_logits=fromLogits, sparse_label=sparseLabel)
if loss == 'softmax_cross_entropy':
fromLogits = loss_params['from_logits'] if 'from_logits' in loss_params else False
loss_function = mx.gluon.loss.SoftmaxCrossEntropyLoss(from_logits=fromLogits, sparse_label=sparseLabel)
ignore_indices = [2]
loss_function = SoftmaxCrossEntropyLossIgnoreIndices(ignore_indices=ignore_indices, from_logits=fromLogits, sparse_label=sparseLabel)
elif loss == 'sigmoid_binary_cross_entropy':
loss_function = mx.gluon.loss.SigmoidBinaryCrossEntropyLoss()
elif loss == 'cross_entropy':
......@@ -309,7 +314,9 @@ class ${tc.fileNameWithoutEnding}:
${input_name} = batch.data[0].as_in_context(mx_context)
</#list>
<#list tc.architectureOutputs as output_name>
<#if output_name != 'attention_'>
${output_name}label = batch.label[${output_name?index}].as_in_context(mx_context)
</#if>
</#list>
outputs=[]
......@@ -350,7 +357,9 @@ class ${tc.fileNameWithoutEnding}:
labels = [
<#list tc.architectureOutputs as output_name>
<#if output_name != 'attention_'>
batch.label[${output_name?index}].as_in_context(mx_context)<#sep>,
</#if>
</#list>
]
......@@ -360,6 +369,10 @@ class ${tc.fileNameWithoutEnding}:
if True: <#-- Fix indentation -->
<#include "pythonExecute.ftl">
<#include "saveAttentionImageTrain.ftl">
predictions = []
for output_name in outputs:
if mx.nd.shape_array(mx.nd.squeeze(output_name)).size > 1:
......@@ -380,7 +393,9 @@ class ${tc.fileNameWithoutEnding}:
labels = [
<#list tc.architectureOutputs as output_name>
<#if output_name != 'attention_'>
batch.label[${output_name?index}].as_in_context(mx_context)<#sep>,
</#if>
</#list>
]
......@@ -390,6 +405,10 @@ class ${tc.fileNameWithoutEnding}:
if True: <#-- Fix indentation -->
<#include "pythonExecute.ftl">
<#include "saveAttentionImageTest.ftl">
predictions = []
for output_name in outputs:
if mx.nd.shape_array(mx.nd.squeeze(output_name)).size > 1:
......
......@@ -36,6 +36,9 @@ if __name__ == "__main__":
<#if (config.normalize)??>
normalize=${config.normalize?string("True","False")},
</#if>
<#if (config.saveAttentionImage)??>
save_attention_image='${config.saveAttentionImage?string("True","False")}',
</#if>
<#if (config.evalMetric)??>
eval_metric='${config.evalMetric.name}',
eval_metric_params={
......
<#-- This template is not used if the following architecture element is an output. See Output.ftl -->
<#if element.axis == -1>
<#assign axis = element.axis?c>
<#else>
<#assign axis = (element.axis + 1)?c>
</#if>
<#assign input = element.inputs[0]>
<#if mode == "FORWARD_FUNCTION">
${element.name} = F.softmax(${input}, axis=${axis})
......
<#list tc.architectureInputSymbols as input>
vector<float> ${tc.getName(input)} = CNNTranslator::translate(${input.name}<#if input.arrayAccess.isPresent()>[${input.arrayAccess.get().intValue.get()?c}]</#if>);
</#list>
<#list tc.getLayerVariableMembers()?keys as member>
vector<float> ${member}(${tc.join(tc.getLayerVariableMembers()[member], " * ")});
<#list tc.getLayerVariableMembers(true)?keys as member>
vector<float> ${member}(${tc.join(tc.getLayerVariableMembers(true)[member], " * ")});
</#list>
<#list tc.architectureOutputSymbols as output>
......
<#list tc.getLayerVariableMembers()?keys as member>
${member} = mx.nd.zeros((batch_size, ${tc.join(tc.cutDimensions(tc.getLayerVariableMembers()[member]), ", ")},), ctx=mx_context)
<#list tc.getLayerVariableMembers(false)?keys as member>
${member} = mx.nd.zeros((batch_size, ${tc.join(tc.cutDimensions(tc.getLayerVariableMembers(false)[member]), ", ")},), ctx=mx_context)
</#list>
<#list tc.architectureOutputSymbols as output>
${tc.getName(output)} = mx.nd.zeros((batch_size, ${tc.join(output.ioDeclaration.type.dimensions, ", ")},), ctx=mx_context)
......@@ -8,6 +8,7 @@
${tc.getName(constant)} = mx.nd.full((batch_size, 1,), ${constant.intValue?c}, ctx=mx_context)
</#list>
attentionList=[]
<#assign instructionCounter = 0>
<#list tc.architecture.networkInstructions as networkInstruction>
<#if networkInstruction.isUnroll()>
......@@ -18,7 +19,12 @@
<#assign width = tc.getBeamSearchWidth(networkInstruction.toUnrollInstruction())>
${tc.getStreamOutputNames(networkInstruction.body, resolvedBody)[0]} = applyBeamSearch(input, 0, ${length}, ${width}, 1.0, ${networkInstruction?index}, input)
<#else>
<#if tc.isAttentionNetwork()>
${tc.join(tc.getStreamOutputNames(networkInstruction.body, resolvedBody), ", ")}, attention_ = self._networks[${networkInstruction?index}](${tc.join(tc.getStreamInputNames(networkInstruction.body, resolvedBody), ", ")})
attentionList.append(attention_)
<#else>
${tc.join(tc.getStreamOutputNames(networkInstruction.body, resolvedBody), ", ")} = self._networks[${networkInstruction?index}](${tc.join(tc.getStreamInputNames(networkInstruction.body, resolvedBody), ", ")})
</#if>
<#if !(tc.getStreamOutputNames(networkInstruction.body, resolvedBody)[0]?ends_with("_output_"))>
outputs.append(${tc.getStreamOutputNames(networkInstruction.body, resolvedBody)[0]})
</#if>
......@@ -42,7 +48,7 @@
</#list>
<#else>
${tc.include(networkInstruction.body, "PYTHON_INLINE")}
<#if !(tc.getStreamOutputNames(networkInstruction.body)[0]?ends_with("_state_"))>
<#if !(tc.getStreamOutputNames(networkInstruction.body)[0]?contains("_state_"))>
outputs.append(${tc.getStreamOutputNames(networkInstruction.body)[0]})
</#if>
</#if>
......
<#list tc.getLayerVariableMembers()?keys as member>
${member} = mx.nd.zeros((batch_size, ${tc.join(tc.cutDimensions(tc.getLayerVariableMembers()[member]), ", ")},), ctx=mx_context)
<#list tc.getLayerVariableMembers(false)?keys as member>
${member} = mx.nd.zeros((batch_size, ${tc.join(tc.cutDimensions(tc.getLayerVariableMembers(false)[member]), ", ")},), ctx=mx_context)
</#list>
<#list tc.architectureOutputSymbols as output>
${tc.getName(output)} = mx.nd.zeros((batch_size, ${tc.join(output.ioDeclaration.type.dimensions, ", ")},), ctx=mx_context)
......@@ -12,13 +12,18 @@
<#list tc.architecture.networkInstructions as networkInstruction>
<#if networkInstruction.isUnroll()>
<#list networkInstruction.toUnrollInstruction().resolvedBodies as resolvedBody>
<#if tc.isAttentionNetwork()>
${tc.join(tc.getStreamOutputNames(networkInstruction.body, resolvedBody), ", ")}, _ = self._networks[${networkInstruction?index}](${tc.join(tc.getStreamInputNames(networkInstruction.body, resolvedBody), ", ")})
<#else>
${tc.join(tc.getStreamOutputNames(networkInstruction.body, resolvedBody), ", ")} = self._networks[${networkInstruction?index}](${tc.join(tc.getStreamInputNames(networkInstruction.body, resolvedBody), ", ")})
</#if>
lossList.append(loss_function(${tc.getStreamOutputNames(networkInstruction.body, resolvedBody)[0]}, ${tc.getStreamOutputNames(networkInstruction.body, resolvedBody)[0]}label))
<#list resolvedBody.elements as element>
<#if element.name == "ArgMax">
${tc.getStreamOutputNames(networkInstruction.body, resolvedBody)[0]} = mx.nd.argmax(${tc.getStreamOutputNames(networkInstruction.body, resolvedBody)[0]}, axis=1).expand_dims(1)
</#if>
</#list>
${tc.getStreamOutputNames(networkInstruction.body, resolvedBody)[0]} = mx.nd.expand_dims(${tc.getStreamOutputNames(networkInstruction.body, resolvedBody)[0]}label, axis=1)
</#list>
<#else>
<#if networkInstruction.body.isTrainable()>
......
if save_attention_image == "True":
plt.clf()
fig = plt.figure(figsize=(10,10))
max_length = len(labels)-1
for l in range(max_length):
attention = attentionList[l]
attention = mx.nd.slice_axis(attention, axis=2, begin=0, end=1)
attention = mx.nd.slice_axis(attention, axis=0, begin=0, end=1)
attention = mx.nd.squeeze(attention)
attention_resized = np.resize(attention.asnumpy(), (8, 8))
ax = fig.add_subplot(max_length//3, max_length//4, l+1)
ax.set_title(dict[int(mx.nd.slice_axis(mx.nd.argmax(outputs[l+1], axis=1), axis=0, begin=0, end=1).asscalar())])
img = ax.imshow(test_images[0+batch_size*(batch_i)])
ax.imshow(attention_resized, cmap='gray', alpha=0.6, extent=img.get_extent())
plt.tight_layout()
plt.savefig(target_dir + '/attention_test.png')
plt.close()
\ No newline at end of file
if save_attention_image == "True":
import matplotlib.pyplot as plt
logging.getLogger('matplotlib').setLevel(logging.ERROR)
plt.clf()
fig = plt.figure(figsize=(10,10))
max_length = len(labels)-1
if(os.path.isfile('src/test/resources/training_data/Show_attend_tell/dict.pkl')):
with open('src/test/resources/training_data/Show_attend_tell/dict.pkl', 'rb') as f:
dict = pickle.load(f)
for l in range(max_length):
attention = attentionList[l]
attention = mx.nd.slice_axis(attention, axis=0, begin=0, end=1)
attention = mx.nd.squeeze(attention)
attention_resized = np.resize(attention.asnumpy(), (8, 8))
ax = fig.add_subplot(max_length//3, max_length//4, l+1)
ax.set_title(dict[int(labels[l+1][0].asscalar())])
img = ax.imshow(train_images[0+batch_size*(batch_i)])
ax.imshow(attention_resized, cmap='gray', alpha=0.6, extent=img.get_extent())
plt.tight_layout()
target_dir = 'target/attention_images'
if not os.path.exists(target_dir):
os.makedirs(target_dir)
plt.savefig(target_dir + '/attention_train.png')
plt.close()
\ No newline at end of file
......@@ -18,12 +18,16 @@ class CNNDataLoader_Alexnet:
train_data = {}
data_mean = {}
data_std = {}
train_images = {}
for input_name in self._input_names_:
train_data[input_name] = train_h5[input_name]
data_mean[input_name + '_'] = nd.array(train_h5[input_name][:].mean(axis=0))
data_std[input_name + '_'] = nd.array(train_h5[input_name][:].std(axis=0) + 1e-5)
if 'images' in train_h5:
train_images = train_h5['images']
train_label = {}
index = 0
for output_name in self._output_names_:
......@@ -38,20 +42,25 @@ class CNNDataLoader_Alexnet:
if test_h5 != None:
test_data = {}
test_images = {}
for input_name in self._input_names_:
test_data[input_name] = test_h5[input_name]
if 'images' in test_h5:
test_images = test_h5['images']
test_label = {}
index = 0
for output_name in self._output_names_:
test_label[index] = test_h5[output_name]
index += 1
test_iter = mx.io.NDArrayIter(data=test_data,
label=test_label,
batch_size=batch_size)
return train_iter, test_iter, data_mean, data_std
return train_iter, test_iter, data_mean, data_std, train_images, test_images
def load_h5_files(self):
train_h5 = None
......
......@@ -18,12 +18,16 @@ class CNNDataLoader_CifarClassifierNetwork:
train_data = {}
data_mean = {}
data_std = {}
train_images = {}
for input_name in self._input_names_:
train_data[input_name] = train_h5[input_name]
data_mean[input_name + '_'] = nd.array(train_h5[input_name][:].mean(axis=0))
data_std[input_name + '_'] = nd.array(train_h5[input_name][:].std(axis=0) + 1e-5)
if 'images' in train_h5:
train_images = train_h5['images']
train_label = {}
index = 0
for output_name in self._output_names_:
......@@ -38,20 +42,25 @@ class CNNDataLoader_CifarClassifierNetwork:
if test_h5 != None:
test_data = {}
test_images = {}
for input_name in self._input_names_:
test_data[input_name] = test_h5[input_name]
if 'images' in test_h5:
test_images = test_h5['images']
test_label = {}
index = 0
for output_name in self._output_names_:
test_label[index] = test_h5[output_name]
index += 1
test_iter = mx.io.NDArrayIter(data=test_data,
label=test_label,
batch_size=batch_size)
return train_iter, test_iter, data_mean, data_std
return train_iter, test_iter, data_mean, data_std, train_images, test_images
def load_h5_files(self):
train_h5 = None
......
......@@ -18,12 +18,16 @@ class CNNDataLoader_VGG16:
train_data = {}
data_mean = {}
data_std = {}
train_images = {}
for input_name in self._input_names_:
train_data[input_name] = train_h5[input_name]
data_mean[input_name + '_'] = nd.array(train_h5[input_name][:].mean(axis=0))
data_std[input_name + '_'] = nd.array(train_h5[input_name][:].std(axis=0) + 1e-5)
if 'images' in train_h5:
train_images = train_h5['images']
train_label = {}
index = 0
for output_name in self._output_names_:
......@@ -38,20 +42,25 @@ class CNNDataLoader_VGG16:
if test_h5 != None:
test_data = {}
test_images = {}
for input_name in self._input_names_:
test_data[input_name] = test_h5[input_name]
if 'images' in test_h5:
test_images = test_h5['images']
test_label = {}
index = 0
for output_name in self._output_names_:
test_label[index] = test_h5[output_name]
index += 1
test_iter = mx.io.NDArrayIter(data=test_data,
label=test_label,
batch_size=batch_size)
return train_iter, test_iter, data_mean, data_std
return train_iter, test_iter, data_mean, data_std, train_images, test_images
def load_h5_files(self):
train_h5 = None
......
......@@ -220,6 +220,7 @@ class CNNSupervisedTrainer_Alexnet:
load_checkpoint=True,
context='gpu',
checkpoint_period=5,
save_attention_image=False,
normalize=True):
if context == 'gpu':
mx_context = mx.gpu()
......@@ -244,7 +245,7 @@ class CNNSupervisedTrainer_Alexnet:
del optimizer_params['learning_rate_decay']
train_iter, test_iter, data_mean, data_std = self._data_loader.load_data(batch_size)
train_iter, test_iter, data_mean, data_std, train_images, test_images = self._data_loader.load_data(batch_size)
if normalize:
self._net_creator.construct(context=mx_context, data_mean=data_mean, data_std=data_std)
......@@ -270,9 +271,13 @@ class CNNSupervisedTrainer_Alexnet:
margin = loss_params['margin'] if 'margin' in loss_params else 1.0
sparseLabel = loss_params['sparse_label'] if 'sparse_label' in loss_params else True
#if loss == 'softmax_cross_entropy':
# fromLogits = loss_params['from_logits'] if 'from_logits' in loss_params else False
# loss_function = mx.gluon.loss.SoftmaxCrossEntropyLoss(from_logits=fromLogits, sparse_label=sparseLabel)
if loss == 'softmax_cross_entropy':
fromLogits = loss_params['from_logits'] if 'from_logits' in loss_params else False
loss_function = mx.gluon.loss.SoftmaxCrossEntropyLoss(from_logits=fromLogits, sparse_label=sparseLabel)
ignore_indices = [2]
loss_function = SoftmaxCrossEntropyLossIgnoreIndices(ignore_indices=ignore_indices, from_logits=fromLogits, sparse_label=sparseLabel)
elif loss == 'sigmoid_binary_cross_entropy':
loss_function = mx.gluon.loss.SigmoidBinaryCrossEntropyLoss()
elif loss == 'cross_entropy':
......@@ -352,12 +357,44 @@ class CNNSupervisedTrainer_Alexnet:
outputs=[]
if True:
if True:
predictions_ = mx.nd.zeros((batch_size, 10,), ctx=mx_context)
attentionList=[]
predictions_ = self._networks[0](data_)
outputs.append(predictions_)
if save_attention_image == "True":
import matplotlib.pyplot as plt
logging.getLogger('matplotlib').setLevel(logging.ERROR)
plt.clf()
fig = plt.figure(figsize=(10,10))
max_length = len(labels)-1
if(os.path.isfile('src/test/resources/training_data/Show_attend_tell/dict.pkl')):
with open('src/test/resources/training_data/Show_attend_tell/dict.pkl', 'rb') as f:
dict = pickle.load(f)
for l in range(max_length):
attention = attentionList[l]
attention = mx.nd.slice_axis(attention, axis=0, begin=0, end=1)
attention = mx.nd.squeeze(attention)
attention_resized = np.resize(attention.asnumpy(), (8, 8))
ax = fig.add_subplot(max_length//3, max_length//4, l+1)
ax.set_title(dict[int(labels[l+1][0].asscalar())])
img = ax.imshow(train_images[0+batch_size*(batch_i)])
ax.imshow(attention_resized, cmap='gray', alpha=0.6, extent=img.get_extent())
plt.tight_layout()
target_dir = 'target/attention_images'
if not os.path.exists(target_dir):
os.makedirs(target_dir)
plt.savefig(target_dir + '/attention_train.png')
plt.close()
predictions = []
for output_name in outputs:
if mx.nd.shape_array(mx.nd.squeeze(output_name)).size > 1:
......@@ -380,12 +417,35 @@ class CNNSupervisedTrainer_Alexnet:
outputs=[]
if True:
if True:
predictions_ = mx.nd.zeros((batch_size, 10,), ctx=mx_context)
attentionList=[]
predictions_ = self._networks[0](data_)
outputs.append(predictions_)
if save_attention_image == "True":
plt.clf()
fig = plt.figure(figsize=(10,10))
max_length = len(labels)-1