Commit f1869937 authored by Julian Dierkes's avatar Julian Dierkes

fixed merge conflicts with master

parents 118babe2 75fd8c2c
......@@ -19,7 +19,7 @@ git masterJobLinux:
integrationMXNetJobLinux:
stage: linux
image: registry.git.rwth-aachen.de/monticore/embeddedmontiarc/generators/emadl2cpp/integrationtests/mxnet:v0.0.3
image: registry.git.rwth-aachen.de/monticore/embeddedmontiarc/generators/emadl2cpp/integrationtests/mxnet:v0.0.4
script:
- mvn -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -B clean install --settings settings.xml -Dtest=IntegrationMXNetTest
......@@ -33,7 +33,7 @@ integrationCaffe2JobLinux:
integrationGluonJobLinux:
stage: linux
image: registry.git.rwth-aachen.de/monticore/embeddedmontiarc/generators/emadl2cpp/integrationtests/mxnet:v0.0.3
image: registry.git.rwth-aachen.de/monticore/embeddedmontiarc/generators/emadl2cpp/integrationtests/mxnet:v0.0.4
script:
- mvn -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -B clean install --settings settings.xml -Dtest=IntegrationGluonTest
......
This diff is collapsed.
This diff is collapsed.
......@@ -9,22 +9,22 @@
<groupId>de.monticore.lang.monticar</groupId>
<artifactId>embedded-montiarc-emadl-generator</artifactId>
<version>0.3.7-SNAPSHOT</version>
<version>0.3.8-SNAPSHOT</version>
<!-- == PROJECT DEPENDENCIES ============================================= -->
<properties>
<!-- .. SE-Libraries .................................................. -->
<emadl.version>0.2.10-SNAPSHOT</emadl.version>
<CNNTrain.version>0.3.8-SNAPSHOT</CNNTrain.version>
<emadl.version>0.2.11-SNAPSHOT</emadl.version>
<CNNTrain.version>0.3.9-SNAPSHOT</CNNTrain.version>
<cnnarch-generator.version>0.0.5-SNAPSHOT</cnnarch-generator.version>
<cnnarch-mxnet-generator.version>0.2.17-SNAPSHOT</cnnarch-mxnet-generator.version>
<cnnarch-caffe2-generator.version>0.2.13-SNAPSHOT</cnnarch-caffe2-generator.version>
<cnnarch-gluon-generator.version>0.2.9-SNAPSHOT</cnnarch-gluon-generator.version>
<cnnarch-caffe2-generator.version>0.2.14-SNAPSHOT</cnnarch-caffe2-generator.version>
<cnnarch-gluon-generator.version>0.2.10-SNAPSHOT</cnnarch-gluon-generator.version>
<cnnarch-tensorflow-generator.version>0.1.0-SNAPSHOT</cnnarch-tensorflow-generator.version>
<Common-MontiCar.version>0.0.14-20180704.113055-2</Common-MontiCar.version>
<embedded-montiarc-math-opt-generator>0.1.5</embedded-montiarc-math-opt-generator>
<embedded-montiarc-emadl-pythonwrapper-generator>0.0.2</embedded-montiarc-emadl-pythonwrapper-generator>
<!-- .. Libraries .................................................. -->
<guava.version>18.0</guava.version>
......@@ -88,6 +88,12 @@
<artifactId>cnnarch-gluon-generator</artifactId>
<version>${cnnarch-gluon-generator.version}</version>
</dependency>
<dependency>
<groupId>de.monticore.lang.monticar</groupId>
<artifactId>common-monticar</artifactId>
<version>${Common-MontiCar.version}</version>
</dependency>
<dependency>
<groupId>de.monticore.lang.monticar</groupId>
......
......@@ -58,7 +58,6 @@ public class IntegrationGluonTest extends IntegrationTest {
assertTrue(Log.getFindings().isEmpty());
}
@Ignore
@Test
public void testShowAttendTell() {
Log.getFindings().clear();
......
......@@ -42,7 +42,6 @@ public class IntegrationTensorflowTest extends IntegrationTest {
super("TENSORFLOW", "39253EC049D4A4E5FA0536AD34874B9D#1DBAEE1B1BD83FB7CB5F70AE91B29638#C4C23549E737A759721D6694C75D9771#5AF0CE68E408E8C1F000E49D72AC214A");
}
@Ignore
@Test
public void testMultipleStreams() {
Log.getFindings().clear();
......
......@@ -3,14 +3,17 @@ FROM maven:3-jdk-8
RUN apt-get update && \
apt-get install -y --no-install-recommends \
git \
libgtk2.0-dev \
python-subprocess32 \
python-tk \
wget python gcc \
build-essential cmake \
liblapack-dev libblas-dev libboost-dev libarmadillo-dev && \
rm -rf /var/lib/apt/lists/*
rm -rf /var/lib/apt/lists/*
RUN git clone https://github.com/apache/incubator-mxnet.git mxnet-source && \
cd mxnet-source && git checkout tags/1.4.0 && cd .. && \
cp -r mxnet-source/include/mxnet /usr/include/mxnet && \
rm -r mxnet-source
RUN wget https://bootstrap.pypa.io/get-pip.py
RUN python get-pip.py
RUN pip install mxnet h5py
RUN pip install mxnet h5py opencv-python matplotlib
package showAttendTell;
component Main{
ports in Z(0:255)^{3, 224, 224} image,
out Z(0:25316)^{1} softmax[19];
ports in Z(0:255)^{3, 224, 224} images,
in Z(-oo:oo)^{64,2048} data,
out Z(0:37758)^{1} target[25];
instance Show_attend_tell net1;
instance Show_attend_tell net;
connect images -> net.images;
connect net.target[:] -> target[:];
}
......@@ -3,7 +3,10 @@ configuration Show_attend_tell{
batch_size:2
context:cpu
eval_metric:bleu
loss:softmax_cross_entropy
loss:softmax_cross_entropy_ignore_indices{
ignore_indices:2
}
use_teacher_forcing:true
save_attention_image:true
optimizer:adam{
learning_rate:0.005
......
......@@ -9,10 +9,10 @@ component Show_attend_tell{
layer LSTM(units=512) decoder;
layer FullyConnected(units = 256) features;
layer FullyConnected(units = 256, flatten=false) features;
layer FullyConnected(units = 1, flatten=false) attention;
1 -> target[0];
0 -> target[0];
data -> features;
......@@ -30,7 +30,6 @@ component Show_attend_tell{
Tanh() ->
FullyConnected(units=1, flatten=false) ->
Softmax(axis=0) ->
Dropout(p=0.25) ->
attention
|
features.output
......@@ -47,7 +46,6 @@ component Show_attend_tell{
FullyConnected(units=37758) ->
Tanh() ->
Dropout(p=0.25) ->
Softmax() ->
ArgMax() ->
target[t]
};
......
......@@ -11,15 +11,15 @@ component Show_attend_tell_images_as_input{
layer LSTM(units=512) decoder;
layer FullyConnected(units = 256) features;
layer FullyConnected(units = 256, flatten=false) features;
layer FullyConnected(units = 1, flatten=false) attention;
1 -> target[0];
0 -> target[0];
images ->
Convolution(kernel=(7,7), channels=64, stride=(7,7), padding="valid") ->
Convolution(kernel=(4,4), channels=64, stride=(4,4), padding="valid") ->
GlobalPooling(pool_type="max") ->
Convolution(kernel=(7,7), channels=128, stride=(7,7), padding="same") ->
Convolution(kernel=(4,4), channels=128, stride=(4,4), padding="same") ->
Reshape(shape=(64, 128)) ->
features;
timed <t> GreedySearch(max_length=25){
......@@ -36,7 +36,6 @@ component Show_attend_tell_images_as_input{
Tanh() ->
FullyConnected(units=1, flatten=false) ->
Softmax(axis=0) ->
Dropout(p=0.25) ->
attention
|
features.output
......@@ -53,7 +52,6 @@ component Show_attend_tell_images_as_input{
FullyConnected(units=37758) ->
Tanh() ->
Dropout(p=0.25) ->
Softmax() ->
ArgMax() ->
target[t]
};
......
import mxnet as mx
import numpy as np
import math
from mxnet import gluon
......@@ -148,16 +147,3 @@ class Net_0(gluon.HybridBlock):
return predictions_
def getInputs(self):
inputs = {}
input_dimensions = (1,28,28)
input_domains = (int,0.0,255.0)
inputs["image_"] = input_domains + (input_dimensions,)
return inputs
def getOutputs(self):
outputs = {}
output_dimensions = (10,1,1)
output_domains = (float,0.0,1.0)
outputs["predictions_"] = output_domains + (output_dimensions,)
return outputs
......@@ -44,7 +44,7 @@ public:
MXPredGetOutputShape(handle, output_index, &shape, &shape_len);
size = 1;
for (mx_uint i = 0; i < shape_len; ++i) size *= shape[i];
assert(out_predictions_.size() == 1 || size == out_predictions_.size());
assert(size == out_predictions_.size());
MXPredGetOutput(handle, output_index, &(out_predictions_[0]), out_predictions_.size());
}
......
......@@ -52,10 +52,9 @@ class SoftmaxCrossEntropyLossIgnoreIndices(gluon.loss.Loss):
else:
label = _reshape_like(F, label, pred)
loss = -(pred * label).sum(axis=self._axis, keepdims=True)
#loss = _apply_weighting(F, loss, self._weight, sample_weight)
# ignore some indices for loss, e.g. <pad> tokens in NLP applications
for i in self._ignore_indices:
loss = loss * mx.nd.logical_not(mx.nd.equal(mx.nd.argmax(pred, axis=1), mx.nd.ones_like(mx.nd.argmax(pred, axis=1))*i))
loss = loss * mx.nd.logical_not(mx.nd.equal(mx.nd.argmax(pred, axis=1), mx.nd.ones_like(mx.nd.argmax(pred, axis=1))*i) * mx.nd.equal(mx.nd.argmax(pred, axis=1), label))
return loss.mean(axis=self._batch_axis, exclude=True)
@mx.metric.register
......@@ -189,6 +188,7 @@ class CNNSupervisedTrainer_mnist_mnistClassifier_net:
context='gpu',
checkpoint_period=5,
save_attention_image=False,
use_teacher_forcing=False,
normalize=True):
if context == 'gpu':
mx_context = mx.gpu()
......@@ -241,12 +241,12 @@ class CNNSupervisedTrainer_mnist_mnistClassifier_net:
margin = loss_params['margin'] if 'margin' in loss_params else 1.0
sparseLabel = loss_params['sparse_label'] if 'sparse_label' in loss_params else True
#if loss == 'softmax_cross_entropy':
# fromLogits = loss_params['from_logits'] if 'from_logits' in loss_params else False
# loss_function = mx.gluon.loss.SoftmaxCrossEntropyLoss(from_logits=fromLogits, sparse_label=sparseLabel)
ignore_indices = [loss_params['ignore_indices']] if 'ignore_indices' in loss_params else []
if loss == 'softmax_cross_entropy':
fromLogits = loss_params['from_logits'] if 'from_logits' in loss_params else False
ignore_indices = [2]
loss_function = mx.gluon.loss.SoftmaxCrossEntropyLoss(from_logits=fromLogits, sparse_label=sparseLabel)
elif loss == 'softmax_cross_entropy_ignore_indices':
fromLogits = loss_params['from_logits'] if 'from_logits' in loss_params else False
loss_function = SoftmaxCrossEntropyLossIgnoreIndices(ignore_indices=ignore_indices, from_logits=fromLogits, sparse_label=sparseLabel)
elif loss == 'sigmoid_binary_cross_entropy':
loss_function = mx.gluon.loss.SigmoidBinaryCrossEntropyLoss()
......@@ -278,6 +278,7 @@ class CNNSupervisedTrainer_mnist_mnistClassifier_net:
tic = None
for epoch in range(begin_epoch, begin_epoch + num_epoch):
train_iter.reset()
for batch_i, batch in enumerate(train_iter):
with autograd.record():
......@@ -288,6 +289,8 @@ class CNNSupervisedTrainer_mnist_mnistClassifier_net:
predictions_ = mx.nd.zeros((train_batch_size, 10,), ctx=mx_context)
nd.waitall()
lossList = []
predictions_ = self._networks[0](image_)
......@@ -318,6 +321,7 @@ class CNNSupervisedTrainer_mnist_mnistClassifier_net:
tic = None
train_test_iter.reset()
metric = mx.metric.create(eval_metric, **eval_metric_params)
for batch_i, batch in enumerate(train_test_iter):
......@@ -329,6 +333,8 @@ class CNNSupervisedTrainer_mnist_mnistClassifier_net:
predictions_ = mx.nd.zeros((test_batch_size, 10,), ctx=mx_context)
nd.waitall()
outputs = []
attentionList=[]
predictions_ = self._networks[0](image_)
......@@ -337,32 +343,43 @@ class CNNSupervisedTrainer_mnist_mnistClassifier_net:
if save_attention_image == "True":
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
logging.getLogger('matplotlib').setLevel(logging.ERROR)
plt.clf()
fig = plt.figure(figsize=(10,10))
fig = plt.figure(figsize=(15,15))
max_length = len(labels)-1
if(os.path.isfile('src/test/resources/training_data/Show_attend_tell/dict.pkl')):
with open('src/test/resources/training_data/Show_attend_tell/dict.pkl', 'rb') as f:
dict = pickle.load(f)
ax = fig.add_subplot(max_length//3, max_length//4, 1)
ax.imshow(train_images[0+test_batch_size*(batch_i)].transpose(1,2,0))
for l in range(max_length):
attention = attentionList[l]
attention = mx.nd.slice_axis(attention, axis=0, begin=0, end=1)
attention = mx.nd.squeeze(attention)
attention = mx.nd.slice_axis(attention, axis=0, begin=0, end=1).squeeze()
attention_resized = np.resize(attention.asnumpy(), (8, 8))
ax = fig.add_subplot(max_length//3, max_length//4, l+1)
ax.set_title(dict[int(labels[l+1][0].asscalar())])
ax = fig.add_subplot(max_length//3, max_length//4, l+2)
if int(labels[l+1][0].asscalar()) > len(dict):
ax.set_title("<unk>")
elif dict[int(labels[l+1][0].asscalar())] == "<end>":
ax.set_title(".")
img = ax.imshow(train_images[0+test_batch_size*(batch_i)].transpose(1,2,0))
ax.imshow(attention_resized, cmap='gray', alpha=0.6, extent=img.get_extent())
break
else:
ax.set_title(dict[int(labels[l+1][0].asscalar())])
img = ax.imshow(train_images[0+test_batch_size*(batch_i)].transpose(1,2,0))
ax.imshow(attention_resized, cmap='gray', alpha=0.6, extent=img.get_extent())
plt.tight_layout()
target_dir = 'target/attention_images'
if not os.path.exists(target_dir):
os.makedirs(target_dir)
os.makedirs(target_dir)
plt.savefig(target_dir + '/attention_train.png')
plt.close()
......@@ -387,6 +404,8 @@ class CNNSupervisedTrainer_mnist_mnistClassifier_net:
predictions_ = mx.nd.zeros((test_batch_size, 10,), ctx=mx_context)
nd.waitall()
outputs = []
attentionList=[]
predictions_ = self._networks[0](image_)
......@@ -396,21 +415,29 @@ class CNNSupervisedTrainer_mnist_mnistClassifier_net:
if save_attention_image == "True":
plt.clf()
fig = plt.figure(figsize=(10,10))
fig = plt.figure(figsize=(15,15))
max_length = len(labels)-1
ax = fig.add_subplot(max_length//3, max_length//4, 1)
ax.imshow(test_images[0+test_batch_size*(batch_i)].transpose(1,2,0))
for l in range(max_length):
attention = attentionList[l]
attention = mx.nd.slice_axis(attention, axis=2, begin=0, end=1)
attention = mx.nd.slice_axis(attention, axis=0, begin=0, end=1)
attention = mx.nd.squeeze(attention)
attention = mx.nd.slice_axis(attention, axis=0, begin=0, end=1).squeeze()
attention_resized = np.resize(attention.asnumpy(), (8, 8))
ax = fig.add_subplot(max_length//3, max_length//4, l+1)
ax.set_title(dict[int(mx.nd.slice_axis(mx.nd.argmax(outputs[l+1], axis=1), axis=0, begin=0, end=1).asscalar())])
ax = fig.add_subplot(max_length//3, max_length//4, l+2)
if int(mx.nd.slice_axis(outputs[l+1], axis=0, begin=0, end=1).squeeze().asscalar()) > len(dict):
ax.set_title("<unk>")
elif dict[int(mx.nd.slice_axis(outputs[l+1], axis=0, begin=0, end=1).squeeze().asscalar())] == "<end>":
ax.set_title(".")
img = ax.imshow(test_images[0+test_batch_size*(batch_i)].transpose(1,2,0))
ax.imshow(attention_resized, cmap='gray', alpha=0.6, extent=img.get_extent())
break
else:
ax.set_title(dict[int(mx.nd.slice_axis(outputs[l+1], axis=0, begin=0, end=1).squeeze().asscalar())])
img = ax.imshow(test_images[0+test_batch_size*(batch_i)].transpose(1,2,0))
ax.imshow(attention_resized, cmap='gray', alpha=0.6, extent=img.get_extent())
plt.tight_layout()
plt.savefig(target_dir + '/attention_test.png')
plt.close()
......
import mxnet as mx
import numpy as np
import math
from mxnet import gluon
......@@ -123,16 +122,3 @@ class Net_0(gluon.HybridBlock):
return qvalues_
def getInputs(self):
inputs = {}
input_dimensions = (4)
input_domains = (float,float('-inf'),float('inf'))
inputs["state_"] = input_domains + (input_dimensions,)
return inputs
def getOutputs(self):
outputs = {}
output_dimensions = (2,1,1)
output_domains = (float,float('-inf'),float('inf'))
outputs["qvalues_"] = output_domains + (output_dimensions,)
return outputs
......@@ -44,7 +44,7 @@ public:
MXPredGetOutputShape(handle, output_index, &shape, &shape_len);
size = 1;
for (mx_uint i = 0; i < shape_len; ++i) size *= shape[i];
assert(out_qvalues_.size() == 1 || size == out_qvalues_.size());
assert(size == out_qvalues_.size());
MXPredGetOutput(handle, output_index, &(out_qvalues_[0]), out_qvalues_.size());
}
......
# (c) https://github.com/MontiCore/monticore
import os
import h5py
import mxnet as mx
import logging
import sys
import numpy as np
import cv2
import importlib
from mxnet import nd
class CNNDataLoader_mountaincar_master_actor:
......@@ -15,197 +13,43 @@ class CNNDataLoader_mountaincar_master_actor:
def __init__(self):
self._data_dir = "data/"
def load_data(self, train_batch_size, test_batch_size):
def load_data(self, batch_size):
train_h5, test_h5 = self.load_h5_files()
train_data = {}
data_mean = {}
data_std = {}
train_images = {}
for input_name in self._input_names_:
train_data[input_name] = train_h5[input_name]
data_mean[input_name + '_'] = nd.array(train_h5[input_name][:].mean(axis=0))
data_std[input_name + '_'] = nd.array(train_h5[input_name][:].std(axis=0) + 1e-5)
if 'images' in train_h5:
train_images = train_h5['images']
data_mean[input_name] = nd.array(train_h5[input_name][:].mean(axis=0))
data_std[input_name] = nd.array(train_h5[input_name][:].std(axis=0) + 1e-5)
train_label = {}
index = 0
for output_name in self._output_names_:
train_label[index] = train_h5[output_name]
index += 1
train_label[output_name] = train_h5[output_name]
train_iter = mx.io.NDArrayIter(data=train_data,
label=train_label,
batch_size=train_batch_size)
train_test_iter = mx.io.NDArrayIter(data=train_data,
label=train_label,
batch_size=test_batch_size)
batch_size=batch_size)
test_iter = None
if test_h5 != None:
test_data = {}
test_images = {}
for input_name in self._input_names_:
test_data[input_name] = test_h5[input_name]
if 'images' in test_h5:
test_images = test_h5['images']
test_label = {}
index = 0
for output_name in self._output_names_:
test_label[index] = test_h5[output_name]
index += 1
test_label[output_name] = test_h5[output_name]
test_iter = mx.io.NDArrayIter(data=test_data,
label=test_label,
batch_size=test_batch_size)
return train_iter, train_test_iter, test_iter, data_mean, data_std, train_images, test_images
def load_data(self, batch_size, img_size):
train_h5, test_h5 = self.load_h5_files()
width = img_size[0]
height = img_size[1]
comb_data = {}
data_mean = {}
data_std = {}
for input_name in self._input_names_:
train_data = train_h5[input_name][:]
test_data = test_h5[input_name][:]
train_shape = train_data.shape
test_shape = test_data.shape
comb_data[input_name] = mx.nd.zeros((train_shape[0]+test_shape[0], train_shape[1], width, height))
for i, img in enumerate(train_data):
img = img.transpose(1,2,0)
comb_data[input_name][i] = cv2.resize(img, (width, height)).reshape((train_shape[1],width,height))
for i, img in enumerate(test_data):
img = img.transpose(1, 2, 0)
comb_data[input_name][i+train_shape[0]] = cv2.resize(img, (width, height)).reshape((train_shape[1], width, height))
data_mean[input_name + '_'] = nd.array(comb_data[input_name][:].mean(axis=0))
data_std[input_name + '_'] = nd.array(comb_data[input_name][:].asnumpy().std(axis=0) + 1e-5)
comb_label = {}
for output_name in self._output_names_:
train_labels = train_h5[output_name][:]
test_labels = test_h5[output_name][:]
comb_label[output_name] = np.append(train_labels, test_labels, axis=0)
train_iter = mx.io.NDArrayIter(data=comb_data,
label=comb_label,
batch_size=batch_size)
test_iter = None
return train_iter, test_iter, data_mean, data_std
def load_preprocessed_data(self, batch_size, preproc_lib):
train_h5, test_h5 = self.load_h5_files()
wrapper = importlib.import_module(preproc_lib)
instance = getattr(wrapper, preproc_lib)()
instance.init()
lib_head, _sep, tail = preproc_lib.rpartition('_')
inp = getattr(wrapper, lib_head + "_input")()
train_data = {}
train_label = {}
data_mean = {}
data_std = {}
shape_output = self.preprocess_data(instance, inp, 0, train_h5)
train_len = len(train_h5[self._input_names_[0]])
for input_name in self._input_names_:
if type(getattr(shape_output, input_name + "_out")) == np.ndarray:
cur_shape = (train_len,) + getattr(shape_output, input_name + "_out").shape
else:
cur_shape = (train_len, 1)
train_data[input_name] = mx.nd.zeros(cur_shape)
for output_name in self._output_names_:
if type(getattr(shape_output, output_name + "_out")) == nd.array:
cur_shape = (train_len,) + getattr(shape_output, output_name + "_out").shape
else:
cur_shape = (train_len, 1)
train_label[output_name] = mx.nd.zeros(cur_shape)
for i in range(train_len):
output = self.preprocess_data(instance, inp, i, train_h5)
for input_name in self._input_names_:
train_data[input_name][i] = getattr(output, input_name + "_out")
for output_name in self._output_names_:
train_label[output_name][i] = getattr(shape_output, output_name + "_out")
for input_name in self._input_names_:
data_mean[input_name + '_'] = nd.array(train_data[input_name][:].mean(axis=0))
data_std[input_name + '_'] = nd.array(train_data[input_name][:].asnumpy().std(axis=0) + 1e-5)
train_iter = mx.io.NDArrayIter(data=train_data,
label=train_label,
batch_size=batch_size)
test_data = {}
test_label = {}
shape_output = self.preprocess_data(instance, inp, 0, test_h5)
test_len = len(test_h5[self._input_names_[0]])
for input_name in self._input_names_:
if type(getattr(shape_output, input_name + "_out")) == np.ndarray:
cur_shape = (test_len,) + getattr(shape_output, input_name + "_out").shape
else:
cur_shape = (test_len, 1)