Commit e32fd9f5 authored by lr119628's avatar lr119628
Browse files

[update] removed AdaNet code that isn't needed anymore

parent 9ee0ef48
......@@ -181,7 +181,7 @@ class ${tc.fileNameWithoutEnding}:
def construct(self, context, data_mean=None, data_std=None):
<#list tc.architecture.networkInstructions as networkInstruction>
<#if tc.containsAdaNet()>
self.networks[${networkInstruction?index}] = Net_${networkInstruction?index}(prefix="",operations=None,batch_size=1)
self.networks[${networkInstruction?index}] = Net_${networkInstruction?index}()
self.dataClass[${networkInstruction?index}] = DataClass_${networkInstruction?index}()
<#else>
self.networks[${networkInstruction?index}] = Net_${networkInstruction?index}(data_mean=data_mean, data_std=data_std, mx_context=context, prefix="")
......
......@@ -528,60 +528,24 @@ class EpisodicMemory(EpisodicReplayMemoryInterface):
self.label_memory.append(mem_dict[key])
<#if tc.containsAdaNet()>
# Generation of the artificial blocks for the Streams below
from mxnet.gluon import nn, HybridBlock
from numpy import log, product
from mxnet.ndarray import zeros
from mxnet.gluon import nn
<#list tc.architecture.networkInstructions as networkInstruction>
<#if networkInstruction.body.containsAdaNet()>
${tc.include(networkInstruction.body, "ADANET_CONSTRUCTION")}
#class Model(gluon.HybridBlock): THIS IS THE ORIGINAL NAME, MUST BE RENAMED IN THE OTHER PARTS
<#assign outblock = networkInstruction.body.getElements()[1].getDeclaration().getBlock("outBlock")>
<#assign block = networkInstruction.body.getElements()[1].getDeclaration().getBlock("block")>
<#assign inblock = networkInstruction.body.getElements()[1].getDeclaration().getBlock("outBlock")>
class Net_${networkInstruction?index}(gluon.HybridBlock):
def __init__(self,operations:dict,batch_size:int,generation=True,**kwargs):
# this is a dummy network during the AdaNet generation it gets overridden
# it is only here so many if tags in the .ftl files can be avoided
def __init__(self,**kwargs):
super(Net_${networkInstruction?index},self).__init__(**kwargs)
self.AdaNet = True
self.op_names = []
self.generation = generation,
self.candidate_complexities = {}
with self.name_scope():
self.batch_size=batch_size
self.data_shape = ${tc.getDefinedOutputDimension()}
self.classes = int(prod(list(self.data_shape)))
if operations is None:
operations={'dummy':nn.Dense(units = 10)}
else:
for name,operation in operations.items():
self.__setattr__(name,operation)
self.op_names.append(name)
self.candidate_complexities[name] = operation.get_complexity()
self.out = nn.Dense(units=self.classes,activation=None,flatten=True)
def get_node_count(self)->int:
count = self.classes
for name in self.op_names:
count += self.__getattribute__(name).count_nodes()
return count
self.AdaNet = True
self.dummy = nn.Dense(units=1)
def hybrid_forward(self, F, x):
res_list = []
for name in self.op_names:
res_list.append(self.__getattribute__(name)(x))
if not res_list:
res_list = [F.identity(x)]
res = tuple(res_list)
y = F.concat(*res, dim=1)
y = self.out(y)
y = F.reshape(y,(1,1,self.batch_size,*self.data_shape))
return y
def get_candidate_complexity(self):
mean_complexity = zeros(len(self.op_names))
for i, name in enumerate(self.op_names):
mean_complexity[i] = self.candidate_complexities[name]
return mean_complexity
def hybrid_forward(self,F,x):
return self.dummy(x)
class DataClass_${networkInstruction?index}:
"""
......@@ -592,8 +556,6 @@ class DataClass_${networkInstruction?index}:
self.candidate_complexities = {}
self.name_ = 'Net_${networkInstruction?index}'
self.AdaNet = True
self.Builder = Builder
self.CandidateHull = CandidateHull
<#if outblock.isPresent()>
self.outBlock = ${tc.include(outblock.get(),"ADANET_CONSTRUCTION")}
<#else>
......@@ -611,8 +573,8 @@ class DataClass_${networkInstruction?index}:
</#if>
#self.block = ${tc.include(block.get(),"ADANET_CONSTRUCTION")}
self.model_shape = ${tc.getDefinedOutputDimension()}
self.BuildingBlock = BuildingBlock
self.model_template = Net_${networkInstruction?index}
#self.BuildingBlock = BuildingBlock
#self.model_template = Net_${networkInstruction?index}
</#if>
</#list>
<#else>
......
......@@ -20,6 +20,7 @@ sys.path.insert(1, '${tc.architecture.getAdaNetUtils()}')
#${tc.architecture.getAdaNetUtils()}
import adaNetUtils
from adaNetUtils import objective_function,calculate_l1,CandidateTrainingloss,AdaLoss,train_candidate,train_model,get_trainer
from adanet import fit
#TODO move adanet Classes Builder,candaita etc in own package
</#if>
try:
......@@ -299,7 +300,7 @@ class BLEU(mx.metric.EvalMetric):
return new_list
<#if tc.containsAdaNet()>
<#if tc.containsAdaNet() && false>
"""
def objective_function(model, data, loss, gamma=.1) -> float:
......@@ -435,7 +436,7 @@ def get_trainer(optimizer: str, parameters: dict, optimizer_params: dict) -> mx.
else:
trainer = mx.gluon.Trainer(parameters, optimizer, optimizer_params)
return trainer
"""
def fit(loss: gluon.loss.Loss,
optimizer: str,
......@@ -452,6 +453,7 @@ def fit(loss: gluon.loss.Loss,
) -> gluon.HybridBlock:
logging.info(f"AdaNet: starting with {epochs} epochs and batch_size:{batch_size} ...")
cg = dataClass.Builder(batch_size=batch_size)
model_template = dataClass.model_template
model_operations = {}
model_score = None
......@@ -537,7 +539,7 @@ def fit(loss: gluon.loss.Loss,
logging.info(round_msg + score_msg)
return model
"""
</#if>
class ${tc.fileNameWithoutEnding}:
......@@ -688,6 +690,7 @@ class ${tc.fileNameWithoutEnding}:
optimizer=optimizer,
epochs=num_epoch,
optimizer_params = optimizer_params,
train_iter = train_iter,
dataLoader = self._data_loader,
dataClass = self._dataClass[${networkInstruction?index}],
shuffle_data=shuffle_data,
......@@ -699,19 +702,12 @@ class ${tc.fileNameWithoutEnding}:
)
logging.info(self._networks[${networkInstruction?index}])
logging.info(f"node count: {self._networks[${networkInstruction?index}].get_node_count()}")
if optimizer == "adamw":
trainers = [mx.gluon.Trainer(network.collect_params(), AdamW.AdamW(**optimizer_params)) for network in self._networks.values() if len(network.collect_params().values()) != 0]
else:
trainers = [mx.gluon.Trainer(network.collect_params(), optimizer, optimizer_params) for network in self._networks.values() if len(network.collect_params().values()) != 0]
</#if>
# update trainers
if optimizer == "adamw":
trainers = [mx.gluon.Trainer(network.collect_params(), AdamW.AdamW(**optimizer_params)) for network in self._networks.values() if len(network.collect_params().values()) != 0]
else:
trainers = [mx.gluon.Trainer(network.collect_params(), optimizer, optimizer_params) for network in self._networks.values() if len(network.collect_params().values()) != 0]
</#list>
</#if>
<#list tc.architecture.networkInstructions as networkInstruction>
......
<#assign input = element.inputs[0]>
<#if mode == "ADANET_CONSTRUCTION">
<#assign outBlock = element.element.getDeclaration().getBlock("outBlock")>
<#assign inBlock = element.element.getDeclaration().getBlock("inBlock")>
......@@ -19,219 +18,4 @@ ${tc.include(inBlock.get(),"ARTIFICIAL_ARCH_CLASS")}
${tc.include(outBlock.get(),"ARTIFICIAL_ARCH_CLASS")}
</#if>
</#if>
<#if !Block.isArtificial()>
class DefaultBlock(gluon.HybridBlock):
""""
default Building Block
"""
def __init__(self, units=20, activation='relu', **kwargs):
super(DefaultBlock, self).__init__(**kwargs)
with self.name_scope():
self.ag = nn.Dense(units=units, activation=activation)
def hybrid_forward(self, F, x):
return self.ag(x)
</#if>
class CandidateHull(gluon.HybridBlock):
"""
the hull which holds the stacked building blocks and is potentially added to the model
"""
def __init__(self,
stack: int,
name: str,
output=None,
<#if outBlock.isPresent()>
<#if outBlock.get().isArtificial()>
output_shape =<#list outBlock.get().outputTypes as type>(${tc.join(type.dimensions, ",")})</#list>,
<#else>
output_shape = None,
</#if>
<#else>
output_shape = None,
</#if>
input=None,
<#if inBlock.isPresent()>
<#if inBlock.get().isArtificial()>
input_shape =<#list inBlock.get().outputTypes as type>(${tc.join(type.dimensions, ",")})</#list>,
<#else>
input_shape = None,
</#if>
<#else>
input_shape = None,
</#if>
model_shape = ${tc.getDefinedOutputDimension()},
block_args=None,
batch_size=None,
rade_aprox=sqrt,
**kwargs):
super(CandidateHull, self).__init__(**kwargs)
self.name_ = name
self.names = []
self.stack = stack
self.rade = None
self.complexity = None
self.node_count = None
self.rade_aprox = rade_aprox
with self.name_scope():
self.batch_size = batch_size
self.output_shape = output_shape
self.input_shape = input_shape
self.model_shape = model_shape
self.classes = prod(list(self.model_shape))
if input:
self.input = input()
else:
self.input = None
if block_args is None:
body = {name + f'{i}': BuildingBlock() for i in range(self.stack)}
else:
body = {name + f'{i}': BuildingBlock(**block_args) for i in range(self.stack)}
for name in body: # add a new operation to the candidate
val = body[name]
self.__setattr__(name=name, value=val)
self.names.append(name)
self.body = body
if output:
self.out = output()
else:
self.out = None
self.finalOut = nn.Dense(units=self.classes, activation=None, flatten=False)
def approximate_rade(self)->float:
"""
approximate the rademacher complexity by the natural logarithm of the number of nodes within the candidate
"""
if self.rade is None:
self.rade = self.rade_aprox(self.count_nodes())
return self.rade
def count_nodes(self)->int:
if self.node_count is None:
oc = 0
for name in self.names:
oc += self.__getattribute__(name).count_nodes()
self.node_count = oc
return self.node_count
def get_complexity(self)->float:
"""
returns the complexity calculated by the training loss function
(alpha * r + beta)* ||w||_L1
r = approximation of the rademacher complexity
"""
return self.complexity
def update_complexity(self, complexity)->None:
self.complexity = complexity
def hybrid_forward(self, F, x, *args, **kwargs):
if self.input:
x = self.input(x)
for name in self.names:
x = self.__getattribute__(name)(x)
if self.out:
x = self.out(x)
x = self.finalOut(x)
return x
class BuildingBlock(gluon.HybridBlock):
"""
the essential building block which gets stacked
"""
def __init__(self,
<#if Block.isArtificial()>
operation = ${tc.include(Block,"ADANET_CONSTRUCTION")},
<#else>
operation = DefaultBlock,
</#if>
**kwargs):
super(BuildingBlock, self).__init__(**kwargs)
self.node_count = None
self.operation = operation() # it is expected that the block doesn't take any paramters
self.oc = None
def count_nodes(self):
oc = 0
params = self.collect_params()
for el in params:
if 'bias' in el:
# each node in a layer has a bias parameter
# => number of nodes = number of bias
oc += params[el].shape[0]
self.node_count = oc
return self.node_count
def hybrid_forward(self, F, x, *args, **kwargs):
return self.operation(x)
class Builder:
"""
the object which generates the new candidates
"""
def __init__(self,batch_size):
self.round = 0
<#if outBlock.isPresent()>
<#if outBlock.get().isArtificial()>
self.output = ${tc.include(outBlock.get(),"ADANET_CONSTRUCTION")}
<#else>
self.output = None
</#if>
<#else>
self.output = None
</#if>
<#if inBlock.isPresent()>
<#if inBlock.get().isArtificial()>
self.input = ${tc.include(inBlock.get(),"ADANET_CONSTRUCTION")}
<#else>
self.input = None
</#if>
<#else>
self.input = None
</#if>
self.batch_size = batch_size
self.pre_stack = 1
self.step = 0
self.block_params = None
self.trainIterator = None
self.candidateHistory = {} # dictionary of already trained candidates
def get_candidates(self)->dict:
"""
:returns tuple of two candidate networks the first is of the same size as the previous the the other is depth+1
"""
candidates = {}
for i in range(self.pre_stack+1):
name = f'candidate{i}round{self.round}'
candidate = CandidateHull(name=name,input=self.input,output=self.output, stack=i+1,
block_args=self.block_params,batch_size=self.batch_size)
# TODO train candidate here
candidates.setdefault(name,candidate)
return candidates
def update(self, up=1):
"""
:param up, increases the stack height by up
:returns None
updates the round count and increases the stack height by up
"""
self.pre_stack += up
self.round += 1
</#if>
\ No newline at end of file
<#assign name = element.element.name>
<#if mode == "ADANET_CONSTRUCTION">
architecture_defined_block_${name}
${name}
</#if>
\ No newline at end of file
......@@ -2,9 +2,9 @@
<#assign name = element.element.name>
<#assign args = element.element.arguments>
<#if mode == "ARTIFICIAL_ARCH_CLASS">
class architecture_defined_block_${name}(gluon.HybridBlock):
class ${name}(gluon.HybridBlock):
def __init__(self, **kwargs):
super(architecture_defined_block_${name}, self).__init__(**kwargs)
super(${name}, self).__init__(**kwargs)
with self.name_scope():
${tc.include(element.element,"ARCHITECTURE_DEFINITION")}
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment