Commit 95a67ac4 authored by Sebastian N.'s avatar Sebastian N.
Browse files

Added exclude to bleu

parent 9e3f0875
Pipeline #195629 failed with stages
in 1 minute and 24 seconds
......@@ -293,29 +293,6 @@ public class GluonConfigurationData extends ConfigurationData {
return environmentParameters.containsKey(ENVIRONMENT_REWARD_TOPIC);
}
private Map<String, Object> getMultiParamEntry(final String key, final String valueName) {
if (!configurationContainsKey(key)) {
return null;
}
Map<String, Object> resultView = new HashMap<>();
MultiParamValueSymbol multiParamValue = (MultiParamValueSymbol)this.getConfiguration().getEntryMap()
.get(key).getValue();
resultView.put(valueName, multiParamValue.getValue());
resultView.putAll(multiParamValue.getParameters());
return resultView;
}
private Boolean configurationContainsKey(final String key) {
return this.getConfiguration().getEntryMap().containsKey(key);
}
private Object retrieveConfigurationEntryValueByKey(final String key) {
return this.getConfiguration().getEntry(key).getValue().getValue();
}
private Map<String, Object> getInputParameterWithName(final String parameterName) {
if (!getRlRewardFunctionParameter().isPresent()
|| !getRlRewardFunctionParameter().get().getTypeOfInputPort(parameterName).isPresent()
......
......@@ -41,7 +41,7 @@ class BLEU(mx.metric.EvalMetric):
def __init__(self, exclude=None, name='bleu', output_names=None, label_names=None):
super(BLEU, self).__init__(name=name, output_names=output_names, label_names=label_names)
self._exclude = exclude or set()
self._exclude = exclude or []
self._match_counts = [0 for _ in range(self.N)]
self._counts = [0 for _ in range(self.N)]
......@@ -186,6 +186,7 @@ class ${tc.fileNameWithoutEnding}:
def train(self, batch_size=64,
num_epoch=10,
eval_metric='acc',
eval_metric_params={},
loss ='softmax_cross_entropy',
loss_params={},
optimizer='adam',
......@@ -315,7 +316,7 @@ class ${tc.fileNameWithoutEnding}:
tic = None
train_iter.reset()
metric = mx.metric.create(eval_metric)
metric = mx.metric.create(eval_metric, **eval_metric_params)
for batch_i, batch in enumerate(train_iter):
<#list tc.architectureInputs as input_name>
${input_name} = batch.data[0].as_in_context(mx_context)
......@@ -345,7 +346,7 @@ class ${tc.fileNameWithoutEnding}:
train_metric_score = metric.get()[1]
test_iter.reset()
metric = mx.metric.create(eval_metric)
metric = mx.metric.create(eval_metric, **eval_metric_params)
for batch_i, batch in enumerate(test_iter):
<#list tc.architectureInputs as input_name>
${input_name} = batch.data[0].as_in_context(mx_context)
......
......@@ -37,7 +37,12 @@ if __name__ == "__main__":
normalize=${config.normalize?string("True","False")},
</#if>
<#if (config.evalMetric)??>
eval_metric='${config.evalMetric}',
eval_metric='${config.evalMetric.metric}',
eval_metric_params={
<#if (config.evalMetric.exclude)??>
'exclude': [<#list config.evalMetric.exclude as value>${value}<#sep>, </#list>],
</#if>
},
</#if>
<#if (config.configuration.loss)??>
loss='${config.lossName}',
......
......@@ -5,6 +5,8 @@ import time
import os
import shutil
import pickle
import math
import sys
from mxnet import gluon, autograd, nd
class CrossEntropyLoss(gluon.loss.Loss):
......@@ -32,6 +34,115 @@ class LogCoshLoss(gluon.loss.Loss):
loss = gluon.loss._apply_weighting(F, loss, self._weight, sample_weight)
return F.mean(loss, axis=self._batch_axis, exclude=True)
@mx.metric.register
class BLEU(mx.metric.EvalMetric):
N = 4
def __init__(self, exclude=None, name='bleu', output_names=None, label_names=None):
super(BLEU, self).__init__(name=name, output_names=output_names, label_names=label_names)
self._exclude = exclude or []
self._match_counts = [0 for _ in range(self.N)]
self._counts = [0 for _ in range(self.N)]
self._size_ref = 0
self._size_hyp = 0
def update(self, labels, preds):
labels, preds = mx.metric.check_label_shapes(labels, preds, True)
new_labels = self._convert(labels)
new_preds = self._convert(preds)
for label, pred in zip(new_labels, new_preds):
reference = [word for word in label if word not in self._exclude]
hypothesis = [word for word in pred if word not in self._exclude]
self._size_ref += len(reference)
self._size_hyp += len(hypothesis)
for n in range(self.N):
reference_ngrams = self._get_ngrams(reference, n + 1)
hypothesis_ngrams = self._get_ngrams(hypothesis, n + 1)
match_count = 0
for ngram in hypothesis_ngrams:
if ngram in reference_ngrams:
reference_ngrams.remove(ngram)
match_count += 1
self._match_counts[n] += match_count
self._counts[n] += len(hypothesis_ngrams)
def get(self):
precisions = [sys.float_info.min for n in range(self.N)]
i = 1
for n in range(self.N):
match_counts = self._match_counts[n]
counts = self._counts[n]
if counts != 0:
if match_counts == 0:
i *= 2
match_counts = 1 / i
precisions[n] = match_counts / counts
bleu = self._get_brevity_penalty() * math.exp(sum(map(math.log, precisions)) / self.N)
return (self.name, bleu)
def calculate(self):
precisions = [sys.float_info.min for n in range(self.N)]
i = 1
for n in range(self.N):
match_counts = self._match_counts[n]
counts = self._counts[n]
if counts != 0:
if match_counts == 0:
i *= 2
match_counts = 1 / i
precisions[n] = match_counts / counts
return self._get_brevity_penalty() * math.exp(sum(map(math.log, precisions)) / self.N)
def _get_brevity_penalty(self):
if self._size_hyp >= self._size_ref:
return 1
else:
return math.exp(1 - (self._size_ref / self._size_hyp))
@staticmethod
def _get_ngrams(sentence, n):
ngrams = []
if len(sentence) >= n:
for i in range(len(sentence) - n + 1):
ngrams.append(sentence[i:i+n])
return ngrams
@staticmethod
def _convert(nd_list):
if len(nd_list) == 0:
return []
new_list = [[] for _ in range(nd_list[0].shape[0])]
for element in nd_list:
for i in range(element.shape[0]):
new_list[i].append(element[i].asscalar())
return new_list
class CNNSupervisedTrainer_Alexnet:
def applyBeamSearch(input, length, width, maxLength, currProb, netIndex, bestOutput):
......@@ -75,6 +186,7 @@ class CNNSupervisedTrainer_Alexnet:
def train(self, batch_size=64,
num_epoch=10,
eval_metric='acc',
eval_metric_params={},
loss ='softmax_cross_entropy',
loss_params={},
optimizer='adam',
......@@ -204,7 +316,7 @@ class CNNSupervisedTrainer_Alexnet:
tic = None
train_iter.reset()
metric = mx.metric.create(eval_metric)
metric = mx.metric.create(eval_metric, **eval_metric_params)
for batch_i, batch in enumerate(train_iter):
data_ = batch.data[0].as_in_context(mx_context)
......@@ -214,7 +326,7 @@ class CNNSupervisedTrainer_Alexnet:
outputs=[]
if True:
if True:
predictions_ = mx.nd.zeros((batch_size, 10,), ctx=mx_context)
predictions_ = self._networks[0](data_)
......@@ -228,40 +340,11 @@ class CNNSupervisedTrainer_Alexnet:
else:
predictions.append(output_name)
#Compute BLEU and NIST Score if data folder contains a dictionary -> NLP dataset
if(os.path.isfile('data/Alexnet/dict.pkl')):
with open('data/Alexnet/dict.pkl', 'rb') as f:
dict = pickle.load(f)
import nltk.translate.bleu_score
import nltk.translate.nist_score
prediction = []
for index in range(batch_size):
sentence = ''
for entry in predictions:
sentence += dict[int(entry[index].asscalar())] + ' '
prediction.append(sentence)
for index in range(batch_size):
sentence = ''
for batchEntry in batch.label:
sentence += dict[int(batchEntry[index].asscalar())] + ' '
print("############################")
print("label: ", sentence)
print("prediction: ", prediction[index])
BLEUscore = nltk.translate.bleu_score.sentence_bleu([sentence], prediction[index])
NISTscore = nltk.translate.nist_score.sentence_nist([sentence], prediction[index])
print("BLEU: ", BLEUscore)
print("NIST: ", NISTscore)
print("############################")
metric.update(preds=predictions, labels=labels)
train_metric_score = metric.get()[1]
test_iter.reset()
metric = mx.metric.create(eval_metric)
metric = mx.metric.create(eval_metric, **eval_metric_params)
for batch_i, batch in enumerate(test_iter):
data_ = batch.data[0].as_in_context(mx_context)
......
......@@ -5,6 +5,8 @@ import time
import os
import shutil
import pickle
import math
import sys
from mxnet import gluon, autograd, nd
class CrossEntropyLoss(gluon.loss.Loss):
......@@ -32,6 +34,115 @@ class LogCoshLoss(gluon.loss.Loss):
loss = gluon.loss._apply_weighting(F, loss, self._weight, sample_weight)
return F.mean(loss, axis=self._batch_axis, exclude=True)
@mx.metric.register
class BLEU(mx.metric.EvalMetric):
N = 4
def __init__(self, exclude=None, name='bleu', output_names=None, label_names=None):
super(BLEU, self).__init__(name=name, output_names=output_names, label_names=label_names)
self._exclude = exclude or []
self._match_counts = [0 for _ in range(self.N)]
self._counts = [0 for _ in range(self.N)]
self._size_ref = 0
self._size_hyp = 0
def update(self, labels, preds):
labels, preds = mx.metric.check_label_shapes(labels, preds, True)
new_labels = self._convert(labels)
new_preds = self._convert(preds)
for label, pred in zip(new_labels, new_preds):
reference = [word for word in label if word not in self._exclude]
hypothesis = [word for word in pred if word not in self._exclude]
self._size_ref += len(reference)
self._size_hyp += len(hypothesis)
for n in range(self.N):
reference_ngrams = self._get_ngrams(reference, n + 1)
hypothesis_ngrams = self._get_ngrams(hypothesis, n + 1)
match_count = 0
for ngram in hypothesis_ngrams:
if ngram in reference_ngrams:
reference_ngrams.remove(ngram)
match_count += 1
self._match_counts[n] += match_count
self._counts[n] += len(hypothesis_ngrams)
def get(self):
precisions = [sys.float_info.min for n in range(self.N)]
i = 1
for n in range(self.N):
match_counts = self._match_counts[n]
counts = self._counts[n]
if counts != 0:
if match_counts == 0:
i *= 2
match_counts = 1 / i
precisions[n] = match_counts / counts
bleu = self._get_brevity_penalty() * math.exp(sum(map(math.log, precisions)) / self.N)
return (self.name, bleu)
def calculate(self):
precisions = [sys.float_info.min for n in range(self.N)]
i = 1
for n in range(self.N):
match_counts = self._match_counts[n]
counts = self._counts[n]
if counts != 0:
if match_counts == 0:
i *= 2
match_counts = 1 / i
precisions[n] = match_counts / counts
return self._get_brevity_penalty() * math.exp(sum(map(math.log, precisions)) / self.N)
def _get_brevity_penalty(self):
if self._size_hyp >= self._size_ref:
return 1
else:
return math.exp(1 - (self._size_ref / self._size_hyp))
@staticmethod
def _get_ngrams(sentence, n):
ngrams = []
if len(sentence) >= n:
for i in range(len(sentence) - n + 1):
ngrams.append(sentence[i:i+n])
return ngrams
@staticmethod
def _convert(nd_list):
if len(nd_list) == 0:
return []
new_list = [[] for _ in range(nd_list[0].shape[0])]
for element in nd_list:
for i in range(element.shape[0]):
new_list[i].append(element[i].asscalar())
return new_list
class CNNSupervisedTrainer_CifarClassifierNetwork:
def applyBeamSearch(input, length, width, maxLength, currProb, netIndex, bestOutput):
......@@ -75,6 +186,7 @@ class CNNSupervisedTrainer_CifarClassifierNetwork:
def train(self, batch_size=64,
num_epoch=10,
eval_metric='acc',
eval_metric_params={},
loss ='softmax_cross_entropy',
loss_params={},
optimizer='adam',
......@@ -204,7 +316,7 @@ class CNNSupervisedTrainer_CifarClassifierNetwork:
tic = None
train_iter.reset()
metric = mx.metric.create(eval_metric)
metric = mx.metric.create(eval_metric, **eval_metric_params)
for batch_i, batch in enumerate(train_iter):
data_ = batch.data[0].as_in_context(mx_context)
......@@ -214,7 +326,7 @@ class CNNSupervisedTrainer_CifarClassifierNetwork:
outputs=[]
if True:
if True:
softmax_ = mx.nd.zeros((batch_size, 10,), ctx=mx_context)
softmax_ = self._networks[0](data_)
......@@ -228,40 +340,11 @@ class CNNSupervisedTrainer_CifarClassifierNetwork:
else:
predictions.append(output_name)
#Compute BLEU and NIST Score if data folder contains a dictionary -> NLP dataset
if(os.path.isfile('data/CifarClassifierNetwork/dict.pkl')):
with open('data/CifarClassifierNetwork/dict.pkl', 'rb') as f:
dict = pickle.load(f)
import nltk.translate.bleu_score
import nltk.translate.nist_score
prediction = []
for index in range(batch_size):
sentence = ''
for entry in predictions:
sentence += dict[int(entry[index].asscalar())] + ' '
prediction.append(sentence)
for index in range(batch_size):
sentence = ''
for batchEntry in batch.label:
sentence += dict[int(batchEntry[index].asscalar())] + ' '
print("############################")
print("label: ", sentence)
print("prediction: ", prediction[index])
BLEUscore = nltk.translate.bleu_score.sentence_bleu([sentence], prediction[index])
NISTscore = nltk.translate.nist_score.sentence_nist([sentence], prediction[index])
print("BLEU: ", BLEUscore)
print("NIST: ", NISTscore)
print("############################")
metric.update(preds=predictions, labels=labels)
train_metric_score = metric.get()[1]
test_iter.reset()
metric = mx.metric.create(eval_metric)
metric = mx.metric.create(eval_metric, **eval_metric_params)
for batch_i, batch in enumerate(test_iter):
data_ = batch.data[0].as_in_context(mx_context)
......
......@@ -5,6 +5,8 @@ import time
import os
import shutil
import pickle
import math
import sys
from mxnet import gluon, autograd, nd
class CrossEntropyLoss(gluon.loss.Loss):
......@@ -32,6 +34,115 @@ class LogCoshLoss(gluon.loss.Loss):
loss = gluon.loss._apply_weighting(F, loss, self._weight, sample_weight)
return F.mean(loss, axis=self._batch_axis, exclude=True)
@mx.metric.register
class BLEU(mx.metric.EvalMetric):
N = 4
def __init__(self, exclude=None, name='bleu', output_names=None, label_names=None):
super(BLEU, self).__init__(name=name, output_names=output_names, label_names=label_names)
self._exclude = exclude or []
self._match_counts = [0 for _ in range(self.N)]
self._counts = [0 for _ in range(self.N)]
self._size_ref = 0
self._size_hyp = 0
def update(self, labels, preds):
labels, preds = mx.metric.check_label_shapes(labels, preds, True)
new_labels = self._convert(labels)
new_preds = self._convert(preds)
for label, pred in zip(new_labels, new_preds):
reference = [word for word in label if word not in self._exclude]
hypothesis = [word for word in pred if word not in self._exclude]
self._size_ref += len(reference)
self._size_hyp += len(hypothesis)
for n in range(self.N):
reference_ngrams = self._get_ngrams(reference, n + 1)
hypothesis_ngrams = self._get_ngrams(hypothesis, n + 1)
match_count = 0
for ngram in hypothesis_ngrams:
if ngram in reference_ngrams:
reference_ngrams.remove(ngram)
match_count += 1
self._match_counts[n] += match_count
self._counts[n] += len(hypothesis_ngrams)
def get(self):
precisions = [sys.float_info.min for n in range(self.N)]
i = 1
for n in range(self.N):
match_counts = self._match_counts[n]
counts = self._counts[n]
if counts != 0:
if match_counts == 0:
i *= 2
match_counts = 1 / i
precisions[n] = match_counts / counts
bleu = self._get_brevity_penalty() * math.exp(sum(map(math.log, precisions)) / self.N)
return (self.name, bleu)
def calculate(self):
precisions = [sys.float_info.min for n in range(self.N)]
i = 1
for n in range(self.N):
match_counts = self._match_counts[n]
counts = self._counts[n]
if counts != 0:
if match_counts == 0:
i *= 2
match_counts = 1 / i
precisions[n] = match_counts / counts
return self._get_brevity_penalty() * math.exp(sum(map(math.log, precisions)) / self.N)
def _get_brevity_penalty(self):
if self._size_hyp >= self._size_ref:
return 1
else:
return math.exp(1 - (self._size_ref / self._size_hyp))
@staticmethod
def _get_ngrams(sentence, n):
ngrams = []
if len(sentence) >= n:
for i in range(len(sentence) - n + 1):
ngrams.append(sentence[i:i+n])
return ngrams