Commit 70d467b3 authored by lr119628's avatar lr119628
Browse files

[update] removed dead code

parent b851d19e
......@@ -297,248 +297,6 @@ class BLEU(mx.metric.EvalMetric):
return new_list
<#if tc.containsAdaNet() && false>
"""
def objective_function(model, data, loss, gamma=.1) -> float:
data.reset()
err_list = []
for batch_i, batch in enumerate(data):
pred = model(batch.data[0])[0][0]
label = batch.label[0]
error = loss(pred, label)
err_list.append(error)
err = concatenate(err_list)
c_complexities = model.get_candidate_complexity()
c_complexities = c_complexities * gamma
objective = err.mean()+ c_complexities.mean()
return objective[0][0]
def calculate_l1(params: dict) -> float:
parameter = params
l1 = None
for key in parameter:
if 'weight' in key:
if l1 is None:
l1 = parameter[key].data().abs().sum()
else:
l1 = add(l1, parameter[key].data().abs().sum())
return l1
class CandidateTrainingloss(Loss):
def __init__(self,
weight=None,
candidate=None,
loss=SigmoidBCELoss,
batch_axis=0,
alpha=.07,
beta=0.0001,
gamma=.1,
**kwargs):
super(CandidateTrainingloss, self).__init__(weight, batch_axis, **kwargs)
self.a = alpha # weight for the rade macher approximation
self.b = beta # fixed added value to the weightend rademacher approximation
self.g = gamma # weight of the combined complexity of L1 and rademacher approximation
self.coreLoss = loss # in template, the loss function is passed initialized!!!!
self.model = candidate # candidate to be trained
# noinspection PyMethodOverriding
def hybrid_forward(self, F, x, label, *args, **kwargs):
l1 = calculate_l1(self.model.collect_params())
# calculate regularization term reg
# reg = (alpha*r)+beta , r = rademacher complexity approximation
rade_term = F.add(F.multiply(self.a, self.model.approximate_rade()), self.b)
reg_term = F.multiply(rade_term, l1)
# save the regularization term, since it is needed in the calculation of the objective function
self.model.update_complexity(reg_term)
# calculate the actual loss and add the regularization term
l = self.coreLoss(x, label)
ad = F.multiply(F.ones(l.shape), reg_term* self.g)
res = F.add(l, ad)
return res
class AdaLoss(Loss):
def __init__(self, weight=None, model=None, loss=SigmoidBCELoss, loss_args=(True,), batch_axis=0, lamb=0.0001,gamma=.1,
beta=.0001,
**kwargs):
super(AdaLoss, self).__init__(weight, batch_axis, **kwargs)
self.g = gamma
self.coreLoss = loss
self.model = model
self.c_complexities = self.model.get_candidate_complexity() # get candidate complexities
self.lamb = lamb
self.beta = beta
def hybrid_forward(self, F, x, label):
cl = self.coreLoss(x, label)
l1 = calculate_l1(self.model.out.collect_params())
reg_term = F.sum(((self.lamb * self.c_complexities) + self.beta) * l1)
return F.add(cl, reg_term*self.g)
def fitComponent(trainIter: mx.io.NDArrayIter, trainer: mx.gluon.Trainer, epochs: int, component: gluon.HybridBlock,
loss_class: gluon.loss, loss_params: dict,model_flag:bool,batch_size:int,log_period=100) -> List[float]:
loss_list = []
loss = loss_class(**loss_params)
for epoch in range(epochs):
trainIter.reset()
for batch_i, batch in enumerate(trainIter):
with autograd.record():
data = batch.data[0]
label = batch.label[0]
if model_flag:
pred = component(data)[0][0]
else:
pred = component(data)
error = loss(pred, label)
error.backward()
trainer.step(data.shape[0], ignore_stale_grad=True)
# if batch_i%log_period==0:
loss_avg = error.mean().asscalar()
loss_list.append(loss_avg)
#logging.info("In Epoch[%d] trained %s with average Loss: %.5f" % (epoch, 'model'if model_flag else component.name_,loss_avg))
return loss_list
def train_candidate(candidate,epochs:int,optimizer:str,optimizer_params:dict,trainIter,loss:Loss,batch_size:int)->List[float]:
candidate_trainer = get_trainer(optimizer,candidate.collect_params(),optimizer_params)
return fitComponent(trainIter=trainIter, trainer=candidate_trainer, epochs=epochs, component=candidate,
loss_class=CandidateTrainingloss, loss_params={'loss': loss, 'candidate': candidate},model_flag=False,batch_size=batch_size)
def train_model(candidate,epochs:int,optimizer:str,optimizer_params:dict,trainIter,loss:Loss,batch_size:int)->List[float]:
params = candidate.out.collect_params()
model_trainer = get_trainer(optimizer, params, optimizer_params)
return fitComponent(trainIter=trainIter, trainer=model_trainer, epochs=epochs, component=candidate,
loss_class=AdaLoss, loss_params={'loss': loss, 'model': candidate},model_flag=True,batch_size=batch_size)
def get_trainer(optimizer: str, parameters: dict, optimizer_params: dict) -> mx.gluon.Trainer:
# gluon.Trainer doesnt take a ctx
if optimizer == 'Adamw':
trainer = mx.gluon.Trainer(parameters, AdamW.AdamW(**optimizer_params) )
else:
trainer = mx.gluon.Trainer(parameters, optimizer, optimizer_params)
return trainer
def fit(loss: gluon.loss.Loss,
optimizer: str,
epochs: int,
optimizer_params: dict,
dataLoader,
dataClass,
shuffle_data: bool,
preprocessing: bool,
T=100,
batch_size=10,
ctx=None,
logging=None
) -> gluon.HybridBlock:
logging.info(f"AdaNet: starting with {epochs} epochs and batch_size:{batch_size} ...")
cg = dataClass.Builder(batch_size=batch_size)
model_template = dataClass.model_template
model_operations = {}
model_score = None
model = model_template(model_operations, batch_size=batch_size)
if ctx is None:
ctx = mx.gpu() if mx.context.num_gpus() else mx.cpu()
model.initialize(ctx=ctx)
if preprocessing:
preproc_lib = "CNNPreprocessor_${tc.fileNameWithoutEnding?keep_after("CNNSupervisedTrainer_")}_executor"
train_iter, test_iter, data_mean, data_std, train_images, test_images = dataLoader.load_preprocessed_data(
batch_size, preproc_lib, shuffle_data)
else:
train_iter, test_iter, data_mean, data_std, train_images, test_images = dataLoader.load_data(batch_size,
shuffle_data)
for rnd in range(T):
# get new candidates
candidates = cg.get_candidates()
can_count = len([x for x in candidates])
model_data = {}
for name,candidate in candidates.items():
model_eval = {}
candidate.initialize(ctx=ctx)
candidate.hybridize()
candidate_loss = train_candidate(candidate,epochs,optimizer,optimizer_params,train_iter,loss,batch_size=batch_size)
model_name = name+ '_model'
# add the current candidate as operation
candidate_op = model_operations.copy()
candidate_op[name] = candidate
# create new model
candidate_model = model_template(operations=candidate_op,batch_size=batch_size)
candidate_model.out.initialize(ctx=ctx)
candidate_model.hybridize()
model_loss = train_model(candidate_model,epochs,optimizer,optimizer_params,train_iter,loss,batch_size=batch_size)
objective_score = objective_function(model=candidate_model, data=train_iter, loss=loss)
model_eval['model'] = candidate_model
model_eval['score'] = objective_score
model_eval['operation'] = candidate
model_data[model_name] = model_eval
logging_msg = "candidate average loss:{:.5f} model average loss:{:.5f} objective score: {:.5f}".format(np.mean(candidate_loss),np.mean(model_loss),objective_score.asscalar())
logging.info(logging_msg)
min_name = None
min_score = None
for name in model_data:
score = model_data[name]['score']
if min_score is None:
min_score = score
min_name = name
elif min_score > score:
min_name = name
min_score = score
model,operation,score = model_data[min_name]['model'],model_data[min_name]['operation'],model_data[min_name]['score']
if model_score is None:
model_score = score
old_score = nd.array(model_score)
else:
# if the new score is better than the old one continue else return current model
if score <= model_score:
old_score = nd.array(model_score)
model_score = nd.array(score)
else:
logging.info("AdaNet: abort in Round {}/{}".format(rnd + 1, T))
# this is not a finally trained model!!
model = model_template(operations=model_operations, generation=False, batch_size=batch_size)
model.hybridize()
model.initialize(ctx=ctx, force_reinit=True)
return model
model_operations[operation.name] = operation
cg.update()
round_msg = 'AdaNet:round: {}/{} finished,'.format(rnd + 1, T)
improvement = (1 - (model_score / old_score).asscalar()) * 100
score_msg = 'current model score:{:.5f} improvement {:.5f}% current model node count:{}'.format(model_score.asscalar(), improvement,model.get_node_count())
logging.info(round_msg + score_msg)
return model
"""
</#if>
class ${tc.fileNameWithoutEnding}:
def __init__(self, data_loader, net_constructor):
self._data_loader = data_loader
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment