Skip to content
GitLab
Commit 2bf1f19f authored by Sebastian Nickels's avatar Sebastian Nickels
Browse files

Updated tests

parent d2ed53ad
Pipeline #226561 failed with stages
in 2 minutes and 23 seconds
Hide whitespace changes
Inline Side-by-side
src/test/resources/target_code/gluon/CNNDataLoader_mnist_mnistClassifier_net.py
View file @ 2bf1f19f
......@@ -5,6 +5,7 @@ import logging
import sys
import numpy as np
import cv2
import importlib
from mxnet import nd
class CNNDataLoader_mnist_mnistClassifier_net:
......@@ -14,7 +15,7 @@ class CNNDataLoader_mnist_mnistClassifier_net:
def __init__(self):
self._data_dir = "data/mnist.LeNetNetwork/"
def load_data(self, batch_size):
def load_data(self, batch_size, shuffle=False):
train_h5, test_h5 = self.load_h5_files()
train_data = {}
......@@ -38,7 +39,8 @@ class CNNDataLoader_mnist_mnistClassifier_net:
train_iter = mx.io.NDArrayIter(data=train_data,
label=train_label,
batch_size=batch_size)
batch_size=batch_size,
shuffle=shuffle)
test_iter = None
......@@ -63,47 +65,108 @@ class CNNDataLoader_mnist_mnistClassifier_net:
return train_iter, test_iter, data_mean, data_std, train_images, test_images
def load_data_img(self, batch_size, img_size):
def load_preprocessed_data(self, batch_size, preproc_lib, shuffle=False):
train_h5, test_h5 = self.load_h5_files()
width = img_size[0]
height = img_size[1]
comb_data = {}
wrapper = importlib.import_module(preproc_lib)
instance = getattr(wrapper, preproc_lib)()
instance.init()
lib_head, _sep, tail = preproc_lib.rpartition('_')
inp = getattr(wrapper, lib_head + "_input")()
train_data = {}
train_label = {}
data_mean = {}
data_std = {}
shape_output = self.preprocess_data(instance, inp, 0, train_h5)
train_len = len(train_h5[self._input_names_[0]])
for input_name in self._input_names_:
if type(getattr(shape_output, input_name + "_out")) == np.ndarray:
cur_shape = (train_len,) + getattr(shape_output, input_name + "_out").shape
else:
cur_shape = (train_len, 1)
train_data[input_name] = mx.nd.zeros(cur_shape)
for output_name in self._output_names_:
if type(getattr(shape_output, output_name + "_out")) == nd.array:
cur_shape = (train_len,) + getattr(shape_output, output_name + "_out").shape
else:
cur_shape = (train_len, 1)
train_label[output_name] = mx.nd.zeros(cur_shape)
for i in range(train_len):
output = self.preprocess_data(instance, inp, i, train_h5)
for input_name in self._input_names_:
train_data[input_name][i] = getattr(output, input_name + "_out")
for output_name in self._output_names_:
train_label[output_name][i] = getattr(shape_output, output_name + "_out")
for input_name in self._input_names_:
train_data = train_h5[input_name][:]
test_data = test_h5[input_name][:]
data_mean[input_name + '_'] = nd.array(train_data[input_name][:].mean(axis=0))
data_std[input_name + '_'] = nd.array(train_data[input_name][:].asnumpy().std(axis=0) + 1e-5)
if 'images' in train_h5:
train_images = train_h5['images']
train_shape = train_data.shape
test_shape = test_data.shape
train_iter = mx.io.NDArrayIter(data=train_data,
label=train_label,
batch_size=batch_size,
shuffle=shuffle)
comb_data[input_name] = mx.nd.zeros((train_shape[0]+test_shape[0], train_shape[1], width, height))
for i, img in enumerate(train_data):
img = img.transpose(1,2,0)
comb_data[input_name][i] = cv2.resize(img, (width, height)).reshape((train_shape[1],width,height))
for i, img in enumerate(test_data):
img = img.transpose(1, 2, 0)
comb_data[input_name][i+train_shape[0]] = cv2.resize(img, (width, height)).reshape((train_shape[1], width, height))
test_data = {}
test_label = {}
data_mean[input_name + '_'] = nd.array(comb_data[input_name][:].mean(axis=0))
data_std[input_name + '_'] = nd.array(comb_data[input_name][:].asnumpy().std(axis=0) + 1e-5)
shape_output = self.preprocess_data(instance, inp, 0, test_h5)
test_len = len(test_h5[self._input_names_[0]])
comb_label = {}
for input_name in self._input_names_:
if type(getattr(shape_output, input_name + "_out")) == np.ndarray:
cur_shape = (test_len,) + getattr(shape_output, input_name + "_out").shape
else:
cur_shape = (test_len, 1)
test_data[input_name] = mx.nd.zeros(cur_shape)
for output_name in self._output_names_:
train_labels = train_h5[output_name][:]
test_labels = test_h5[output_name][:]
comb_label[output_name] = np.append(train_labels, test_labels, axis=0)
if type(getattr(shape_output, output_name + "_out")) == nd.array:
cur_shape = (test_len,) + getattr(shape_output, output_name + "_out").shape
else:
cur_shape = (test_len, 1)
test_label[output_name] = mx.nd.zeros(cur_shape)
for i in range(test_len):
output = self.preprocess_data(instance, inp, i, test_h5)
for input_name in self._input_names_:
test_data[input_name][i] = getattr(output, input_name + "_out")
for output_name in self._output_names_:
test_label[output_name][i] = getattr(shape_output, output_name + "_out")
if 'images' in test_h5:
test_images = test_h5['images']
train_iter = mx.io.NDArrayIter(data=comb_data,
label=comb_label,
test_iter = mx.io.NDArrayIter(data=test_data,
label=test_label,
batch_size=batch_size)
test_iter = None
return train_iter, test_iter, data_mean, data_std, train_images, test_images
return train_iter, test_iter, data_mean, data_std
def preprocess_data(self, instance_wrapper, input_wrapper, index, data_h5):
for input_name in self._input_names_:
data = data_h5[input_name][0]
attr = getattr(input_wrapper, input_name)
if (type(data)) == np.ndarray:
data = np.asfortranarray(data).astype(attr.dtype)
else:
data = type(attr)(data)
setattr(input_wrapper, input_name, data)
for output_name in self._output_names_:
data = data_h5[output_name][0]
attr = getattr(input_wrapper, output_name)
if (type(data)) == np.ndarray:
data = np.asfortranarray(data).astype(attr.dtype)
else:
data = type(attr)(data)
setattr(input_wrapper, output_name, data)
return instance_wrapper.execute(input_wrapper)
def load_h5_files(self):
train_h5 = None
......
src/test/resources/target_code/gluon/CNNNet_mnist_mnistClassifier_net.py
View file @ 2bf1f19f
import mxnet as mx
import numpy as np
import math
from mxnet import gluon
......@@ -51,10 +52,10 @@ class Reshape(gluon.HybridBlock):
class CustomRNN(gluon.HybridBlock):
def __init__(self, hidden_size, num_layers, bidirectional, **kwargs):
def __init__(self, hidden_size, num_layers, dropout, bidirectional, **kwargs):
super(CustomRNN, self).__init__(**kwargs)
with self.name_scope():
self.rnn = gluon.rnn.RNN(hidden_size=hidden_size, num_layers=num_layers,
self.rnn = gluon.rnn.RNN(hidden_size=hidden_size, num_layers=num_layers, dropout=dropout,
bidirectional=bidirectional, activation='tanh', layout='NTC')
def hybrid_forward(self, F, data, state0):
......@@ -63,10 +64,10 @@ class CustomRNN(gluon.HybridBlock):
class CustomLSTM(gluon.HybridBlock):
def __init__(self, hidden_size, num_layers, bidirectional, **kwargs):
def __init__(self, hidden_size, num_layers, dropout, bidirectional, **kwargs):
super(CustomLSTM, self).__init__(**kwargs)
with self.name_scope():
self.lstm = gluon.rnn.LSTM(hidden_size=hidden_size, num_layers=num_layers,
self.lstm = gluon.rnn.LSTM(hidden_size=hidden_size, num_layers=num_layers, dropout=dropout,
bidirectional=bidirectional, layout='NTC')
def hybrid_forward(self, F, data, state0, state1):
......@@ -75,10 +76,10 @@ class CustomLSTM(gluon.HybridBlock):
class CustomGRU(gluon.HybridBlock):
def __init__(self, hidden_size, num_layers, bidirectional, **kwargs):
def __init__(self, hidden_size, num_layers, dropout, bidirectional, **kwargs):
super(CustomGRU, self).__init__(**kwargs)
with self.name_scope():
self.gru = gluon.rnn.GRU(hidden_size=hidden_size, num_layers=num_layers,
self.gru = gluon.rnn.GRU(hidden_size=hidden_size, num_layers=num_layers, dropout=dropout,
bidirectional=bidirectional, layout='NTC')
def hybrid_forward(self, F, data, state0):
......@@ -147,3 +148,16 @@ class Net_0(gluon.HybridBlock):
return predictions_
def getInputs(self):
inputs = {}
input_dimensions = (1,28,28)
input_domains = (int,0.0,255.0)
inputs["image_"] = input_domains + (input_dimensions,)
return inputs
def getOutputs(self):
outputs = {}
output_dimensions = (10,1,1)
output_domains = (float,0.0,1.0)
outputs["predictions_"] = output_domains + (output_dimensions,)
return outputs
src/test/resources/target_code/gluon/CNNSupervisedTrainer_mnist_mnistClassifier_net.py
View file @ 2bf1f19f
......@@ -191,7 +191,10 @@ class CNNSupervisedTrainer_mnist_mnistClassifier_net:
context='gpu',
save_attention_image=False,
use_teacher_forcing=False,
normalize=True):
normalize=True,
shuffle_data=False,
clip_global_grad_norm=None,
preprocessing = False):
if context == 'gpu':
mx_context = mx.gpu()
elif context == 'cpu':
......@@ -199,6 +202,12 @@ class CNNSupervisedTrainer_mnist_mnistClassifier_net:
else:
logging.error("Context argument is '" + context + "'. Only 'cpu' and 'gpu are valid arguments'.")
if preprocessing:
preproc_lib = "CNNPreprocessor_mnist_mnistClassifier_net_executor"
train_iter, test_iter, data_mean, data_std, train_images, test_images = self._data_loader.load_preprocessed_data(batch_size, preproc_lib, shuffle_data)
else:
train_iter, test_iter, data_mean, data_std, train_images, test_images = self._data_loader.load_data(batch_size, shuffle_data)
if 'weight_decay' in optimizer_params:
optimizer_params['wd'] = optimizer_params['weight_decay']
del optimizer_params['weight_decay']
......@@ -214,8 +223,6 @@ class CNNSupervisedTrainer_mnist_mnistClassifier_net:
del optimizer_params['step_size']
del optimizer_params['learning_rate_decay']
train_iter, test_iter, data_mean, data_std, train_images, test_images = self._data_loader.load_data(batch_size)
if normalize:
self._net_creator.construct(context=mx_context, data_mean=data_mean, data_std=data_std)
else:
......@@ -223,7 +230,7 @@ class CNNSupervisedTrainer_mnist_mnistClassifier_net:
begin_epoch = 0
if load_checkpoint:
begin_epoch = self._net_creator.load(mx_context)
begin_epoch = self._net_creator.load(mx_context) + 1
else:
if os.path.isdir(self._net_creator._model_dir_):
shutil.rmtree(self._net_creator._model_dir_)
......@@ -276,6 +283,15 @@ class CNNSupervisedTrainer_mnist_mnistClassifier_net:
tic = None
for epoch in range(begin_epoch, begin_epoch + num_epoch):
if shuffle_data:
if preprocessing:
preproc_lib = "CNNPreprocessor_mnist_mnistClassifier_net_executor"
train_iter, test_iter, data_mean, data_std, train_images, test_images = self._data_loader.load_preprocessed_data(batch_size, preproc_lib, shuffle_data)
else:
train_iter, test_iter, data_mean, data_std, train_images, test_images = self._data_loader.load_data(batch_size, shuffle_data)
global_loss_train = 0.0
train_batches = 0
loss_total = 0
train_iter.reset()
......@@ -304,6 +320,17 @@ class CNNSupervisedTrainer_mnist_mnistClassifier_net:
loss_total += loss.sum().asscalar()
global_loss_train += float(loss.mean().asscalar())
train_batches += 1
if clip_global_grad_norm:
grads = []
for network in self._networks.values():
grads.extend([param.grad(mx_context) for param in network.collect_params().values()])
gluon.utils.clip_global_norm(grads, clip_global_grad_norm)
for trainer in trainers:
trainer.step(batch_size)
......@@ -323,6 +350,9 @@ class CNNSupervisedTrainer_mnist_mnistClassifier_net:
tic = time.time()
if train_batches > 0:
global_loss_train /= train_batches
tic = None
......@@ -340,10 +370,12 @@ class CNNSupervisedTrainer_mnist_mnistClassifier_net:
nd.waitall()
outputs = []
attentionList=[]
lossList = []
attentionList = []
predictions_ = self._networks[0](image_)
outputs.append(predictions_)
lossList.append(loss_function(predictions_, labels[0]))
if save_attention_image == "True":
......@@ -399,6 +431,9 @@ class CNNSupervisedTrainer_mnist_mnistClassifier_net:
else:
train_metric_score = 0
global_loss_test = 0.0
test_batches = 0
test_iter.reset()
metric = mx.metric.create(eval_metric, **eval_metric_params)
for batch_i, batch in enumerate(test_iter):
......@@ -413,10 +448,12 @@ class CNNSupervisedTrainer_mnist_mnistClassifier_net:
nd.waitall()
outputs = []
attentionList=[]
lossList = []
attentionList = []
predictions_ = self._networks[0](image_)
outputs.append(predictions_)
lossList.append(loss_function(predictions_, labels[0]))
if save_attention_image == "True":
......@@ -460,6 +497,12 @@ class CNNSupervisedTrainer_mnist_mnistClassifier_net:
os.makedirs(target_dir)
plt.savefig(target_dir + '/attention_test.png')
plt.close()
loss = 0
for element in lossList:
loss = loss + element
global_loss_test += float(loss.mean().asscalar())
test_batches += 1
predictions = []
for output_name in outputs:
......@@ -472,8 +515,10 @@ class CNNSupervisedTrainer_mnist_mnistClassifier_net:
metric.update(preds=predictions, labels=labels)
test_metric_score = metric.get()[1]
logging.info("Epoch[%d] Train: %f, Test: %f" % (epoch, train_metric_score, test_metric_score))
if test_batches > 0:
global_loss_test /= test_batches
logging.info("Epoch[%d] Train metric: %f, Test metric: %f, Train loss: %f, Test loss: %f" % (epoch, train_metric_score, test_metric_score, global_loss_train, global_loss_test))
if (epoch - begin_epoch) % checkpoint_period == 0:
for i, network in self._networks.items():
......
src/test/resources/target_code/gluon/reinforcementModel/cartpole/CNNNet_cartpole_master_dqn.py
View file @ 2bf1f19f
import mxnet as mx
import numpy as np
import math
from mxnet import gluon
......@@ -51,10 +52,10 @@ class Reshape(gluon.HybridBlock):
class CustomRNN(gluon.HybridBlock):
def __init__(self, hidden_size, num_layers, bidirectional, **kwargs):
def __init__(self, hidden_size, num_layers, dropout, bidirectional, **kwargs):
super(CustomRNN, self).__init__(**kwargs)
with self.name_scope():
self.rnn = gluon.rnn.RNN(hidden_size=hidden_size, num_layers=num_layers,
self.rnn = gluon.rnn.RNN(hidden_size=hidden_size, num_layers=num_layers, dropout=dropout,
bidirectional=bidirectional, activation='tanh', layout='NTC')
def hybrid_forward(self, F, data, state0):
......@@ -63,10 +64,10 @@ class CustomRNN(gluon.HybridBlock):
class CustomLSTM(gluon.HybridBlock):
def __init__(self, hidden_size, num_layers, bidirectional, **kwargs):
def __init__(self, hidden_size, num_layers, dropout, bidirectional, **kwargs):
super(CustomLSTM, self).__init__(**kwargs)
with self.name_scope():
self.lstm = gluon.rnn.LSTM(hidden_size=hidden_size, num_layers=num_layers,
self.lstm = gluon.rnn.LSTM(hidden_size=hidden_size, num_layers=num_layers, dropout=dropout,
bidirectional=bidirectional, layout='NTC')
def hybrid_forward(self, F, data, state0, state1):
......@@ -75,10 +76,10 @@ class CustomLSTM(gluon.HybridBlock):
class CustomGRU(gluon.HybridBlock):
def __init__(self, hidden_size, num_layers, bidirectional, **kwargs):
def __init__(self, hidden_size, num_layers, dropout, bidirectional, **kwargs):
super(CustomGRU, self).__init__(**kwargs)
with self.name_scope():
self.gru = gluon.rnn.GRU(hidden_size=hidden_size, num_layers=num_layers,
self.gru = gluon.rnn.GRU(hidden_size=hidden_size, num_layers=num_layers, dropout=dropout,
bidirectional=bidirectional, layout='NTC')
def hybrid_forward(self, F, data, state0):
......@@ -122,3 +123,16 @@ class Net_0(gluon.HybridBlock):
return qvalues_
def getInputs(self):
inputs = {}
input_dimensions = (4)
input_domains = (float,0,1)
inputs["state_"] = input_domains + (input_dimensions,)
return inputs
def getOutputs(self):
outputs = {}
output_dimensions = (2,1,1)
output_domains = (float,float('-inf'),float('inf'))
outputs["qvalues_"] = output_domains + (output_dimensions,)
return outputs
src/test/resources/target_code/gluon/reinforcementModel/mountaincar/CNNNet_mountaincar_master_actor.py
View file @ 2bf1f19f
import mxnet as mx
import numpy as np
import math
from mxnet import gluon
......@@ -51,10 +52,10 @@ class Reshape(gluon.HybridBlock):
class CustomRNN(gluon.HybridBlock):
def __init__(self, hidden_size, num_layers, bidirectional, **kwargs):
def __init__(self, hidden_size, num_layers, dropout, bidirectional, **kwargs):
super(CustomRNN, self).__init__(**kwargs)
with self.name_scope():
self.rnn = gluon.rnn.RNN(hidden_size=hidden_size, num_layers=num_layers,
self.rnn = gluon.rnn.RNN(hidden_size=hidden_size, num_layers=num_layers, dropout=dropout,
bidirectional=bidirectional, activation='tanh', layout='NTC')
def hybrid_forward(self, F, data, state0):
......@@ -63,10 +64,10 @@ class CustomRNN(gluon.HybridBlock):
class CustomLSTM(gluon.HybridBlock):
def __init__(self, hidden_size, num_layers, bidirectional, **kwargs):
def __init__(self, hidden_size, num_layers, dropout, bidirectional, **kwargs):
super(CustomLSTM, self).__init__(**kwargs)
with self.name_scope():
self.lstm = gluon.rnn.LSTM(hidden_size=hidden_size, num_layers=num_layers,
self.lstm = gluon.rnn.LSTM(hidden_size=hidden_size, num_layers=num_layers, dropout=dropout,
bidirectional=bidirectional, layout='NTC')
def hybrid_forward(self, F, data, state0, state1):
......@@ -75,10 +76,10 @@ class CustomLSTM(gluon.HybridBlock):
class CustomGRU(gluon.HybridBlock):
def __init__(self, hidden_size, num_layers, bidirectional, **kwargs):
def __init__(self, hidden_size, num_layers, dropout, bidirectional, **kwargs):
super(CustomGRU, self).__init__(**kwargs)
with self.name_scope():
self.gru = gluon.rnn.GRU(hidden_size=hidden_size, num_layers=num_layers,
self.gru = gluon.rnn.GRU(hidden_size=hidden_size, num_layers=num_layers, dropout=dropout,
bidirectional=bidirectional, layout='NTC')
def hybrid_forward(self, F, data, state0):
......@@ -124,3 +125,16 @@ class Net_0(gluon.HybridBlock):
return action_
def getInputs(self):
inputs = {}
input_dimensions = (2)
input_domains = (float,0,1)
inputs["state_"] = input_domains + (input_dimensions,)
return inputs
def getOutputs(self):
outputs = {}
output_dimensions = (1,1,1)
output_domains = (float,-1.0,1.0)
outputs["action_"] = output_domains + (output_dimensions,)
return outputs
src/test/resources/target_code/gluon/reinforcementModel/mountaincar/reinforcement_learning/CNNNet_mountaincar_agent_mountaincarCritic.py
View file @ 2bf1f19f
import mxnet as mx
import numpy as np
import math
from mxnet import gluon
......@@ -51,10 +52,10 @@ class Reshape(gluon.HybridBlock):
class CustomRNN(gluon.HybridBlock):
def __init__(self, hidden_size, num_layers, bidirectional, **kwargs):
def __init__(self, hidden_size, num_layers, dropout, bidirectional, **kwargs):
super(CustomRNN, self).__init__(**kwargs)
with self.name_scope():
self.rnn = gluon.rnn.RNN(hidden_size=hidden_size, num_layers=num_layers,
self.rnn = gluon.rnn.RNN(hidden_size=hidden_size, num_layers=num_layers, dropout=dropout,
bidirectional=bidirectional, activation='tanh', layout='NTC')
def hybrid_forward(self, F, data, state0):
......@@ -63,10 +64,10 @@ class CustomRNN(gluon.HybridBlock):
class CustomLSTM(gluon.HybridBlock):
def __init__(self, hidden_size, num_layers, bidirectional, **kwargs):
def __init__(self, hidden_size, num_layers, dropout, bidirectional, **kwargs):
super(CustomLSTM, self).__init__(**kwargs)
with self.name_scope():
self.lstm = gluon.rnn.LSTM(hidden_size=hidden_size, num_layers=num_layers,
self.lstm = gluon.rnn.LSTM(hidden_size=hidden_size, num_layers=num_layers, dropout=dropout,
bidirectional=bidirectional, layout='NTC')
def hybrid_forward(self, F, data, state0, state1):
......@@ -75,10 +76,10 @@ class CustomLSTM(gluon.HybridBlock):
class CustomGRU(gluon.HybridBlock):
def __init__(self, hidden_size, num_layers, bidirectional, **kwargs):
def __init__(self, hidden_size, num_layers, dropout, bidirectional, **kwargs):
super(CustomGRU, self).__init__(**kwargs)
with self.name_scope():
self.gru = gluon.rnn.GRU(hidden_size=hidden_size, num_layers=num_layers,
self.gru = gluon.rnn.GRU(hidden_size=hidden_size, num_layers=num_layers, dropout=dropout,
bidirectional=bidirectional, layout='NTC')
def hybrid_forward(self, F, data, state0):
......@@ -135,3 +136,19 @@ class Net_0(gluon.HybridBlock):
return qvalues_
def getInputs(self):
inputs = {}
input_dimensions = (2)
input_domains = (float,0,1)
inputs["state_"] = input_domains + (input_dimensions,)
input_dimensions = (1)
input_domains = (float,-1.0,1.0)
inputs["action_"] = input_domains + (input_dimensions,)