Skip to content

GitLab

Commit eb892398 authored by Nicola Gatto's avatar Nicola Gatto
Browse files

Adapt tests to new version

parent 7c707a98
Hide whitespace changes
Inline Side-by-side
Showing with 209 additions and 195 deletions
pom.xml
View file @ eb892398
......@@ -19,7 +19,7 @@
<CNNTrain.version>0.2.6</CNNTrain.version>
<cnnarch-mxnet-generator.version>0.2.14-SNAPSHOT</cnnarch-mxnet-generator.version>
<cnnarch-caffe2-generator.version>0.2.11-SNAPSHOT</cnnarch-caffe2-generator.version>
<cnnarch-gluon-generator.version>0.1.5</cnnarch-gluon-generator.version>
<cnnarch-gluon-generator.version>0.1.6</cnnarch-gluon-generator.version>
<embedded-montiarc-math-opt-generator>0.1.4</embedded-montiarc-math-opt-generator>
<!-- .. Libraries .................................................. -->
......
src/test/java/de/monticore/lang/monticar/emadl/GenerationTest.java
View file @ eb892398
......@@ -177,6 +177,8 @@ public class GenerationTest extends AbstractSymtabTest {
"mnist_mnistClassifier.h",
"CNNCreator_mnist_mnistClassifier_net.py",
"CNNPredictor_mnist_mnistClassifier_net.h",
"CNNDataLoader_mnist_mnistClassifier_net.py",
"supervised_trainer.py",
"mnist_mnistClassifier_net.h",
"HelperA.h",
"CNNTranslator.h",
......
src/test/java/de/monticore/lang/monticar/emadl/IntegrationTest.java
View file @ eb892398
......@@ -160,7 +160,4 @@ public abstract class IntegrationTest extends AbstractSymtabTest {
deleteHashFile();
}
}
src/test/resources/target_code/gluon/CNNCreator_mnist_mnistClassifier_net.py
View file @ eb892398
import mxnet as mx
import logging
import os
import errno
import shutil
import h5py
import sys
import numpy as np
import time
from mxnet import gluon, autograd, nd
from CNNNet_mnist_mnistClassifier_net import Net
@mx.init.register
class MyConstant(mx.init.Initializer):
def __init__(self, value):
super(MyConstant, self).__init__(value=value)
self.value = value
def _init_weight(self, _, arr):
arr[:] = mx.nd.array(self.value)
class CNNCreator_mnist_mnistClassifier_net:
_data_dir_ = "data/mnist.LeNetNetwork/"
_model_dir_ = "model/mnist.LeNetNetwork/"
_model_prefix_ = "model"
_input_names_ = ['image']
_input_shapes_ = [(1,28,28)]
_output_names_ = ['predictions_label']
def __init__(self):
self.weight_initializer = mx.init.Normal()
......@@ -60,176 +41,6 @@ class CNNCreator_mnist_mnistClassifier_net:
return lastEpoch
def load_data(self, batch_size):
train_h5, test_h5 = self.load_h5_files()
data_mean = train_h5[self._input_names_[0]][:].mean(axis=0)
data_std = train_h5[self._input_names_[0]][:].std(axis=0) + 1e-5
train_iter = mx.io.NDArrayIter(train_h5[self._input_names_[0]],
train_h5[self._output_names_[0]],
batch_size=batch_size,
data_name=self._input_names_[0],
label_name=self._output_names_[0])
test_iter = None
if test_h5 != None:
test_iter = mx.io.NDArrayIter(test_h5[self._input_names_[0]],
test_h5[self._output_names_[0]],
batch_size=batch_size,
data_name=self._input_names_[0],
label_name=self._output_names_[0])
return train_iter, test_iter, data_mean, data_std
def load_h5_files(self):
train_h5 = None
test_h5 = None
train_path = self._data_dir_ + "train.h5"
test_path = self._data_dir_ + "test.h5"
if os.path.isfile(train_path):
train_h5 = h5py.File(train_path, 'r')
if not (self._input_names_[0] in train_h5 and self._output_names_[0] in train_h5):
logging.error("The HDF5 file '" + os.path.abspath(train_path) + "' has to contain the datasets: "
+ "'" + self._input_names_[0] + "', '" + self._output_names_[0] + "'")
sys.exit(1)
test_iter = None
if os.path.isfile(test_path):
test_h5 = h5py.File(test_path, 'r')
if not (self._input_names_[0] in test_h5 and self._output_names_[0] in test_h5):
logging.error("The HDF5 file '" + os.path.abspath(test_path) + "' has to contain the datasets: "
+ "'" + self._input_names_[0] + "', '" + self._output_names_[0] + "'")
sys.exit(1)
else:
logging.warning("Couldn't load test set. File '" + os.path.abspath(test_path) + "' does not exist.")
return train_h5, test_h5
else:
logging.error("Data loading failure. File '" + os.path.abspath(train_path) + "' does not exist.")
sys.exit(1)
def train(self, batch_size=64,
num_epoch=10,
eval_metric='acc',
optimizer='adam',
optimizer_params=(('learning_rate', 0.001),),
load_checkpoint=True,
context='gpu',
checkpoint_period=5,
normalize=True):
if context == 'gpu':
mx_context = mx.gpu()
elif context == 'cpu':
mx_context = mx.cpu()
else:
logging.error("Context argument is '" + context + "'. Only 'cpu' and 'gpu are valid arguments'.")
if 'weight_decay' in optimizer_params:
optimizer_params['wd'] = optimizer_params['weight_decay']
del optimizer_params['weight_decay']
if 'learning_rate_decay' in optimizer_params:
min_learning_rate = 1e-08
if 'learning_rate_minimum' in optimizer_params:
min_learning_rate = optimizer_params['learning_rate_minimum']
del optimizer_params['learning_rate_minimum']
optimizer_params['lr_scheduler'] = mx.lr_scheduler.FactorScheduler(
optimizer_params['step_size'],
factor=optimizer_params['learning_rate_decay'],
stop_factor_lr=min_learning_rate)
del optimizer_params['step_size']
del optimizer_params['learning_rate_decay']
train_iter, test_iter, data_mean, data_std = self.load_data(batch_size)
if self.net == None:
if normalize:
self.construct(context=mx_context, data_mean=nd.array(data_mean), data_std=nd.array(data_std))
else:
self.construct(context=mx_context)
begin_epoch = 0
if load_checkpoint:
begin_epoch = self.load(mx_context)
else:
if os.path.isdir(self._model_dir_):
shutil.rmtree(self._model_dir_)
try:
os.makedirs(self._model_dir_)
except OSError:
if not os.path.isdir(self._model_dir_):
raise
trainer = mx.gluon.Trainer(self.net.collect_params(), optimizer, optimizer_params)
if self.net.last_layer == 'softmax':
loss_function = mx.gluon.loss.SoftmaxCrossEntropyLoss()
elif self.net.last_layer == 'sigmoid':
loss_function = mx.gluon.loss.SigmoidBinaryCrossEntropyLoss()
elif self.net.last_layer == 'linear':
loss_function = mx.gluon.loss.L2Loss()
else: # TODO: Change default?
loss_function = mx.gluon.loss.L2Loss()
logging.warning("Invalid last_layer, defaulting to L2 loss")
speed_period = 50
tic = None
for epoch in range(begin_epoch, begin_epoch + num_epoch):
train_iter.reset()
for batch_i, batch in enumerate(train_iter):
data = batch.data[0].as_in_context(mx_context)
label = batch.label[0].as_in_context(mx_context)
with autograd.record():
output = self.net(data)
loss = loss_function(output, label)
loss.backward()
trainer.step(batch_size)
if tic is None:
tic = time.time()
else:
if batch_i % speed_period == 0:
try:
speed = speed_period * batch_size / (time.time() - tic)
except ZeroDivisionError:
speed = float("inf")
logging.info("Epoch[%d] Batch[%d] Speed: %.2f samples/sec" % (epoch, batch_i, speed))
tic = time.time()
tic = None
train_iter.reset()
metric = mx.metric.create(eval_metric)
for batch_i, batch in enumerate(train_iter):
data = batch.data[0].as_in_context(mx_context)
label = batch.label[0].as_in_context(mx_context)
output = self.net(data)
predictions = mx.nd.argmax(output, axis=1)
metric.update(preds=predictions, labels=label)
train_metric_score = metric.get()[1]
test_iter.reset()
metric = mx.metric.create(eval_metric)
for batch_i, batch in enumerate(test_iter):
data = batch.data[0].as_in_context(mx_context)
label = batch.label[0].as_in_context(mx_context)
output = self.net(data)
predictions = mx.nd.argmax(output, axis=1)
metric.update(preds=predictions, labels=label)
test_metric_score = metric.get()[1]
logging.info("Epoch[%d] Train: %f, Test: %f" % (epoch, train_metric_score, test_metric_score))
if (epoch - begin_epoch) % checkpoint_period == 0:
self.net.save_parameters(self._model_dir_ + self._model_prefix_ + '-' + str(epoch).zfill(4) + '.params')
self.net.save_parameters(self._model_dir_ + self._model_prefix_ + '-'
+ str(num_epoch + begin_epoch).zfill(4) + '.params')
self.net.export(self._model_dir_ + self._model_prefix_ + '_newest', epoch=0)
def construct(self, context, data_mean=None, data_std=None):
self.net = Net(data_mean=data_mean, data_std=data_std)
self.net.collect_params().initialize(self.weight_initializer, ctx=context)
......
src/test/resources/target_code/gluon/CNNDataLoader_mnist_mnistClassifier_net.py 0 → 100644
View file @ eb892398
import os
import h5py
import mxnet as mx
import logging
import sys
class mnist_mnistClassifier_netDataLoader:
_input_names_ = ['image']
_output_names_ = ['predictions_label']
def __init__(self):
self._data_dir = "data/mnist.LeNetNetwork/"
def load_data(self, batch_size):
train_h5, test_h5 = self.load_h5_files()
data_mean = train_h5[self._input_names_[0]][:].mean(axis=0)
data_std = train_h5[self._input_names_[0]][:].std(axis=0) + 1e-5
train_iter = mx.io.NDArrayIter(train_h5[self._input_names_[0]],
train_h5[self._output_names_[0]],
batch_size=batch_size,
data_name=self._input_names_[0],
label_name=self._output_names_[0])
test_iter = None
if test_h5 != None:
test_iter = mx.io.NDArrayIter(test_h5[self._input_names_[0]],
test_h5[self._output_names_[0]],
batch_size=batch_size,
data_name=self._input_names_[0],
label_name=self._output_names_[0])
return train_iter, test_iter, data_mean, data_std
def load_h5_files(self):
train_h5 = None
test_h5 = None
train_path = self._data_dir + "train.h5"
test_path = self._data_dir + "test.h5"
if os.path.isfile(train_path):
train_h5 = h5py.File(train_path, 'r')
if not (self._input_names_[0] in train_h5 and self._output_names_[0] in train_h5):
logging.error("The HDF5 file '" + os.path.abspath(train_path) + "' has to contain the datasets: "
+ "'" + self._input_names_[0] + "', '" + self._output_names_[0] + "'")
sys.exit(1)
test_iter = None
if os.path.isfile(test_path):
test_h5 = h5py.File(test_path, 'r')
if not (self._input_names_[0] in test_h5 and self._output_names_[0] in test_h5):
logging.error("The HDF5 file '" + os.path.abspath(test_path) + "' has to contain the datasets: "
+ "'" + self._input_names_[0] + "', '" + self._output_names_[0] + "'")
sys.exit(1)
else:
logging.warning("Couldn't load test set. File '" + os.path.abspath(test_path) + "' does not exist.")
return train_h5, test_h5
else:
logging.error("Data loading failure. File '" + os.path.abspath(train_path) + "' does not exist.")
sys.exit(1)
\ No newline at end of file
src/test/resources/target_code/gluon/CNNTrainer_mnist_mnistClassifier_net.py
View file @ eb892398
import logging
import mxnet as mx
import supervised_trainer
import CNNCreator_mnist_mnistClassifier_net
import CNNDataLoader_mnist_mnistClassifier_net
if __name__ == "__main__":
logging.basicConfig(level=logging.DEBUG)
......@@ -8,8 +10,12 @@ if __name__ == "__main__":
handler = logging.FileHandler("train.log", "w", encoding=None, delay="true")
logger.addHandler(handler)
mnist_mnistClassifier_net = CNNCreator_mnist_mnistClassifier_net.CNNCreator_mnist_mnistClassifier_net()
mnist_mnistClassifier_net.train(
mnist_mnistClassifier_net_creator = CNNCreator_mnist_mnistClassifier_net.CNNCreator_mnist_mnistClassifier_net()
mnist_mnistClassifier_net_loader = CNNDataLoader_mnist_mnistClassifier_net.mnist_mnistClassifier_netDataLoader()
mnist_mnistClassifier_net_trainer = supervised_trainer.CNNSupervisedTrainer(mnist_mnistClassifier_net_loader,
mnist_mnistClassifier_net_creator)
mnist_mnistClassifier_net_trainer.train(
batch_size=64,
num_epoch=11,
context='gpu',
......
src/test/resources/target_code/gluon/supervised_trainer.py 0 → 100644
View file @ eb892398
import mxnet as mx
import logging
import numpy as np
import time
import os
import shutil
from mxnet import gluon, autograd, nd
class CNNSupervisedTrainer(object):
def __init__(self, data_loader, net_constructor, net=None):
self._data_loader = data_loader
self._net_creator = net_constructor
self._net = net
def train(self, batch_size=64,
num_epoch=10,
eval_metric='acc',
optimizer='adam',
optimizer_params=(('learning_rate', 0.001),),
load_checkpoint=True,
context='gpu',
checkpoint_period=5,
normalize=True):
if context == 'gpu':
mx_context = mx.gpu()
elif context == 'cpu':
mx_context = mx.cpu()
else:
logging.error("Context argument is '" + context + "'. Only 'cpu' and 'gpu are valid arguments'.")
if 'weight_decay' in optimizer_params:
optimizer_params['wd'] = optimizer_params['weight_decay']
del optimizer_params['weight_decay']
if 'learning_rate_decay' in optimizer_params:
min_learning_rate = 1e-08
if 'learning_rate_minimum' in optimizer_params:
min_learning_rate = optimizer_params['learning_rate_minimum']
del optimizer_params['learning_rate_minimum']
optimizer_params['lr_scheduler'] = mx.lr_scheduler.FactorScheduler(
optimizer_params['step_size'],
factor=optimizer_params['learning_rate_decay'],
stop_factor_lr=min_learning_rate)
del optimizer_params['step_size']
del optimizer_params['learning_rate_decay']
train_iter, test_iter, data_mean, data_std = self._data_loader.load_data(batch_size)
if self._net is None:
if normalize:
self._net_creator.construct(
context=mx_context, data_mean=nd.array(data_mean), data_std=nd.array(data_std))
else:
self._net_creator.construct(context=mx_context)
begin_epoch = 0
if load_checkpoint:
begin_epoch = self._net_creator.load(mx_context)
else:
if os.path.isdir(self._net_creator._model_dir_):
shutil.rmtree(self._net_creator._model_dir_)
self._net = self._net_creator.net
try:
os.makedirs(self._net_creator._model_dir_)
except OSError:
if not os.path.isdir(self._net_creator._model_dir_):
raise
trainer = mx.gluon.Trainer(self._net.collect_params(), optimizer, optimizer_params)
if self._net.last_layer == 'softmax':
loss_function = mx.gluon.loss.SoftmaxCrossEntropyLoss()
elif self._net.last_layer == 'sigmoid':
loss_function = mx.gluon.loss.SigmoidBinaryCrossEntropyLoss()
elif self._net.last_layer == 'linear':
loss_function = mx.gluon.loss.L2Loss()
else: # TODO: Change default?
loss_function = mx.gluon.loss.L2Loss()
logging.warning("Invalid last_layer, defaulting to L2 loss")
speed_period = 50
tic = None
for epoch in range(begin_epoch, begin_epoch + num_epoch):
train_iter.reset()
for batch_i, batch in enumerate(train_iter):
data = batch.data[0].as_in_context(mx_context)
label = batch.label[0].as_in_context(mx_context)
with autograd.record():
output = self._net(data)
loss = loss_function(output, label)
loss.backward()
trainer.step(batch_size)
if tic is None:
tic = time.time()
else:
if batch_i % speed_period == 0:
try:
speed = speed_period * batch_size / (time.time() - tic)
except ZeroDivisionError:
speed = float("inf")
logging.info("Epoch[%d] Batch[%d] Speed: %.2f samples/sec" % (epoch, batch_i, speed))
tic = time.time()
tic = None
train_iter.reset()
metric = mx.metric.create(eval_metric)
for batch_i, batch in enumerate(train_iter):
data = batch.data[0].as_in_context(mx_context)
label = batch.label[0].as_in_context(mx_context)
output = self._net(data)
predictions = mx.nd.argmax(output, axis=1)
metric.update(preds=predictions, labels=label)
train_metric_score = metric.get()[1]
test_iter.reset()
metric = mx.metric.create(eval_metric)
for batch_i, batch in enumerate(test_iter):
data = batch.data[0].as_in_context(mx_context)
label = batch.label[0].as_in_context(mx_context)
output = self._net(data)
predictions = mx.nd.argmax(output, axis=1)
metric.update(preds=predictions, labels=label)
test_metric_score = metric.get()[1]
logging.info("Epoch[%d] Train: %f, Test: %f" % (epoch, train_metric_score, test_metric_score))
if (epoch - begin_epoch) % checkpoint_period == 0:
self._net.save_parameters(self.parameter_path() + '-' + str(epoch).zfill(4) + '.params')
self._net.save_parameters(self.parameter_path() + '-' + str(num_epoch + begin_epoch).zfill(4) + '.params')
self._net.export(self.parameter_path() + '_newest', epoch=0)
def parameter_path(self):
return self._net_creator._model_dir_ + self._net_creator._model_prefix_
\ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment