Commit 87eb6d5a authored by lr119628's avatar lr119628
Browse files

[update] restructuring reset point

parent e2013174
......@@ -535,7 +535,9 @@ from mxnet.ndarray import zeros
<#if networkInstruction.body.containsAdaNet()>
${tc.include(networkInstruction.body, "ADANET_CONSTRUCTION")}
#class Model(gluon.HybridBlock): THIS IS THE ORIGINAL NAME, MUST BE RENAMED IN THE OTHER PARTS
<#assign outblock = networkInstruction.body.getElements()[1].getDeclaration().getBlock("outBlock")>
<#assign block = networkInstruction.body.getElements()[1].getDeclaration().getBlock("block")>
<#assign inblock = networkInstruction.body.getElements()[1].getDeclaration().getBlock("outBlock")>
class Net_${networkInstruction?index}(gluon.HybridBlock):
def __init__(self,operations:dict,batch_size:int,generation=True,**kwargs):
super(Net_${networkInstruction?index},self).__init__(**kwargs)
......@@ -543,17 +545,9 @@ class Net_${networkInstruction?index}(gluon.HybridBlock):
self.op_names = []
self.generation = generation,
self.candidate_complexities = {}
<#assign outblock = networkInstruction.body.getElements()[1].getDeclaration().getBlock("outBlock")>
with self.name_scope():
self.batch_size=batch_size
<#if outblock.isPresent()>
self.fout = ${tc.include(outblock.get(),"ADANET_CONSTRUCTION")}
<#else>
self.fout = None
</#if>
self.finalout = None
#if operations is None:
# operations={'dummy':nn.Dense(units = 10)}
self.data_shape = ${tc.getDefinedOutputDimension()}
self.classes = int(prod(list(self.data_shape)))
if operations is None:
......@@ -564,9 +558,7 @@ class Net_${networkInstruction?index}(gluon.HybridBlock):
self.op_names.append(name)
self.candidate_complexities[name] = operation.get_complexity()
self.out = nn.Dense(units=self.classes,activation=None,flatten=True)
if self.fout:
#self.finalout = self.fout()
pass
def get_node_count(self)->int:
count = self.classes
for name in self.op_names:
......@@ -582,8 +574,6 @@ class Net_${networkInstruction?index}(gluon.HybridBlock):
res = tuple(res_list)
y = F.concat(*res, dim=1)
y = self.out(y)
if self.finalout:
y = self.finalout(y)
y = F.reshape(y,(1,1,self.batch_size,*self.data_shape))
return y
......@@ -604,6 +594,22 @@ class DataClass_${networkInstruction?index}:
self.AdaNet = True
self.Builder = Builder
self.CandidateHull = CandidateHull
<#if outblock.isPresent()>
self.outBlock = ${tc.include(outblock.get(),"ADANET_CONSTRUCTION")}
<#else>
self.outBlock = None
</#if>
<#if inblock.isPresent()>
self.inBlock = ${tc.include(inblock.get(),"ADANET_CONSTRUCTION")}
<#else>
self.inBlock = None
</#if>
<#if block.isPresent()>
self.block = ${tc.include(block.get(),"ADANET_CONSTRUCTION")}
<#else>
self.block = None
</#if>
#self.block = ${tc.include(block.get(),"ADANET_CONSTRUCTION")}
self.BuildingBlock = BuildingBlock
self.output_shape = self.CandidateHull(name='getOutputShape',stack=0).output_shape
self.model_template = Net_${networkInstruction?index}
......
......@@ -16,6 +16,11 @@ from mxnet import gluon, autograd, nd
from typing import List
from mxnet.gluon.loss import Loss, SigmoidBCELoss
from mxnet.ndarray import add, concatenate
sys.path.insert(1, '${tc.architecture.getAdaNetUtils()}')
#${tc.architecture.getAdaNetUtils()}
import adaNetUtils
from adaNetUtils import objective_function,calculate_l1,CandidateTrainingloss,AdaLoss,train_candidate,train_model,get_trainer
#TODO move adanet Classes Builder,candaita etc in own package
</#if>
try:
import AdamW
......@@ -295,15 +300,9 @@ class BLEU(mx.metric.EvalMetric):
return new_list
<#if tc.containsAdaNet()>
"""
def objective_function(model, data, loss, gamma=.1) -> float:
"""
:param model:
:param trainer:
:param data:
:param loss:
:param gamma:
:return:
"""
data.reset()
err_list = []
for batch_i, batch in enumerate(data):
......@@ -318,11 +317,8 @@ def objective_function(model, data, loss, gamma=.1) -> float:
return objective[0][0]
def calculate_l1(params: dict) -> float:
"""
calculate the L1 Norm on the weights of the passed model
"""
parameter = params
l1 = None
for key in parameter:
......@@ -333,7 +329,6 @@ def calculate_l1(params: dict) -> float:
l1 = add(l1, parameter[key].data().abs().sum())
return l1
class CandidateTrainingloss(Loss):
def __init__(self,
weight=None,
......@@ -344,13 +339,7 @@ class CandidateTrainingloss(Loss):
beta=0.0001,
gamma=.1,
**kwargs):
"""
loss function which is used to train each candidate
:param loss, can be any (binary) loss function, to the result a regularization term is
added which consists of complexity of the candidate and the L1-Norm applied
to the candidate weights
"""
super(CandidateTrainingloss, self).__init__(weight, batch_axis, **kwargs)
self.a = alpha # weight for the rade macher approximation
......@@ -382,9 +371,7 @@ class CandidateTrainingloss(Loss):
class AdaLoss(Loss):
"""
objective function of the whole model
"""
def __init__(self, weight=None, model=None, loss=SigmoidBCELoss, loss_args=(True,), batch_axis=0, lamb=0.0001,gamma=.1,
beta=.0001,
......@@ -406,11 +393,7 @@ class AdaLoss(Loss):
def fitComponent(trainIter: mx.io.NDArrayIter, trainer: mx.gluon.Trainer, epochs: int, component: gluon.HybridBlock,
loss_class: gluon.loss, loss_params: dict,model_flag:bool,batch_size:int,log_period=100) -> List[float]:
"""
function trains a component of the generated model.
expects a compoment, a trainern instance with corresponding parameters.
"""
loss_list = []
loss = loss_class(**loss_params)
for epoch in range(epochs):
......@@ -440,8 +423,6 @@ def train_candidate(candidate,epochs:int,optimizer:str,optimizer_params:dict,tra
def train_model(candidate,epochs:int,optimizer:str,optimizer_params:dict,trainIter,loss:Loss,batch_size:int)->List[float]:
params = candidate.out.collect_params()
if candidate.finalout is not None:
params.update(candidate.finalout.collect_params())
model_trainer = get_trainer(optimizer, params, optimizer_params)
return fitComponent(trainIter=trainIter, trainer=model_trainer, epochs=epochs, component=candidate,
loss_class=AdaLoss, loss_params={'loss': loss, 'model': candidate},model_flag=True,batch_size=batch_size)
......@@ -454,7 +435,7 @@ def get_trainer(optimizer: str, parameters: dict, optimizer_params: dict) -> mx.
else:
trainer = mx.gluon.Trainer(parameters, optimizer, optimizer_params)
return trainer
"""
def fit(loss: gluon.loss.Loss,
optimizer: str,
......@@ -508,8 +489,7 @@ def fit(loss: gluon.loss.Loss,
candidate_model = model_template(operations=candidate_op,batch_size=batch_size)
candidate_model.out.initialize(ctx=ctx)
if candidate_model.finalout:
candidate_model.finalout.initialize(ctx=ctx)
candidate_model.hybridize()
model_loss = train_model(candidate_model,epochs,optimizer,optimizer_params,train_iter,loss,batch_size=batch_size)
......
......@@ -208,6 +208,9 @@ class Builder:
self.pre_stack = 1
self.step = 0
self.block_params = None
self.trainIterator = None
self.candidateHistory = {} # dictionary of already trained candidates
def get_candidates(self)->dict:
"""
:returns tuple of two candidate networks the first is of the same size as the previous the the other is depth+1
......@@ -217,20 +220,9 @@ class Builder:
name = f'candidate{i}round{self.round}'
candidate = CandidateHull(name=name,input=self.input,output=self.output, stack=i+1,
block_args=self.block_params,batch_size=self.batch_size)
# TODO train candidate here
candidates.setdefault(name,candidate)
"""
c0_name = f'candidate0round{self.round}'
c1_name = f'candidate1round{self.round}'
c0 = CandidateHull(name=c0_name,input=self.input,output=self.output, stack=self.pre_stack,
block_args=self.block_params,batch_size=self.batch_size)
c1 = CandidateHull(name=c1_name,input=self.input,output=self.output ,stack=self.pre_stack + 1,
block_args=self.block_params,batch_size=self.batch_size)
self.step += 1
return {c0.name_:c0, c1.name_:c1}
"""
return candidates
def update(self, up=1):
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment