...
 
Commits (165)
......@@ -53,6 +53,16 @@ def softmax(z):
assert z_flat.ndim == 2
return T.reshape(T.nnet.softmax(z_flat), z.shape)
def log_softmax(z):
assert z.ndim >= 1
if z.ndim <= 2:
return T.nnet.logsoftmax(z)
else:
from TheanoUtil import time_batch_make_flat
z_flat = time_batch_make_flat(z)
assert z_flat.ndim == 2
return T.reshape(T.nnet.logsoftmax(z_flat), z.shape)
def gauss(z):
return T.exp(-T.sqr(z))
......@@ -120,6 +130,7 @@ ActivationFunctions = {
'cos': T.cos,
'complex_bound': complex_bound,
'softmax': softmax,
'log_softmax': log_softmax,
'gauss': gauss,
"erf": T.erf,
"exp": T.exp,
......
......@@ -54,6 +54,7 @@ class Dataset(object):
set_or_remove("seq_ordering", config.value("batching", None))
set_or_remove("shuffle_frames_of_nseqs", config.int('shuffle_frames_of_nseqs', 0) or None)
set_or_remove("min_chunk_size", config.int('min_chunk_size', 0) or None)
set_or_remove("chunking_variance", config.float("chunking_variance", 0))
@staticmethod
def get_default_kwargs_eval(config):
......@@ -85,13 +86,13 @@ class Dataset(object):
window=1, context_window=None, chunking=None,
seq_ordering='default', partition_epoch=None, repeat_epoch=None,
seq_list_filter_file=None, unique_seq_tags=False,
shuffle_frames_of_nseqs=0, min_chunk_size=0,
shuffle_frames_of_nseqs=0, min_chunk_size=0, chunking_variance=0,
estimated_num_seqs=None):
"""
:param str name: e.g. "train" or "eval"
:param int window: features will be of dimension window * feature_dim, as we add a context-window around.
not all datasets support this option.
:param None|int|dict|NumbersDict context_window: will add this context for each chunk
:param None|int|dict|NumbersDict|(dict,dict) context_window: will add this context for each chunk
:param None|str|int|(int,int)|dict|(dict,dict) chunking: "chunk_size:chunk_step"
:param str seq_ordering: "batching"-option in config. e.g. "default", "sorted" or "random".
See self.get_seq_order_for_epoch() for more details.
......@@ -128,6 +129,7 @@ class Dataset(object):
self._num_seqs = 0
self._estimated_num_seqs = estimated_num_seqs
self.min_chunk_size = min_chunk_size
self.chunking_variance = chunking_variance
if isinstance(chunking, str):
if ":" in chunking:
chunking = tuple(map(int, chunking.split(":")))
......@@ -150,14 +152,31 @@ class Dataset(object):
assert sorted(chunk_step.keys()) == sorted(chunk_size.keys())
assert chunk_step.max_value() > 0, "chunking step must be positive (for some key)"
self.chunk_step = chunk_step
if context_window is None:
context_window = NumbersDict(0)
elif isinstance(context_window, int):
context_window = NumbersDict(broadcast_value=0, numbers_dict={"data": context_window})
elif isinstance(context_window, dict):
context_window = NumbersDict(broadcast_value=0, numbers_dict=context_window)
assert isinstance(context_window, NumbersDict)
self.context_window = context_window
if isinstance(context_window, (tuple, list)):
assert len(context_window) == 2
for elem in context_window:
assert isinstance(elem, dict)
# assume that first element of tuple|list is for left context and second element for right context
self.ctx_left = NumbersDict(numbers_dict=context_window[0])
self.ctx_right = NumbersDict(numbers_dict=context_window[1])
else:
if context_window is None:
context_window = NumbersDict()
elif isinstance(context_window, int):
context_window = NumbersDict(numbers_dict={"data": context_window})
elif isinstance(context_window, dict):
context_window = NumbersDict(numbers_dict=context_window)
assert isinstance(context_window, NumbersDict)
# ctx_total is how much frames we add additionally.
# One less because the original frame also counts, and context_window=1 means that we just have that single frame.
ctx_total = NumbersDict.max([context_window, 1]) - 1
# In case ctx_total is odd / context_window is even, we have to decide where to put one more frame.
# To keep it consistent with e.g. 1D convolution with a kernel of even size, we add one more to the right.
# See test_tfconv1d_evensize().
self.ctx_left = ctx_total // 2
self.ctx_right = ctx_total - self.ctx_left
assert isinstance(self.ctx_left, NumbersDict)
assert isinstance(self.ctx_right, NumbersDict)
self.shuffle_frames_of_nseqs = shuffle_frames_of_nseqs
self.epoch = None
......@@ -849,6 +868,9 @@ class Dataset(object):
chunk_step = self.chunk_step
chunk_size = NumbersDict(chunk_size)
chunk_step = NumbersDict(chunk_step)
chunk_size_orig = chunk_size.copy()
chunk_step_orig = chunk_step.copy()
s = 0
while self.is_less_than_num_seqs(s):
length = self.get_seq_length(s)
......@@ -863,6 +885,11 @@ class Dataset(object):
if chunk_step[default_key] == 0: # allow some keys with zero chunk-step
assert chunk_step.max_value() > 0
default_key = [key for key in sorted(used_data_keys) if chunk_step[key] > 0][0]
if self.chunking_variance > 0:
chunking_variance = 1. - self.rnd_seq_drop.random() * self.chunking_variance
for k in used_data_keys:
chunk_size[k] = max(int(chunk_size_orig[k] * chunking_variance), 1)
chunk_step[k] = max(int(chunk_step_orig[k] * chunking_variance), 1)
assert chunk_step[default_key] > 0
t = NumbersDict.constant_like(0, numbers_dict=length)
# There are usually the 'data' (input) and 'classes' (targets) data-keys in `length` but there can be others.
......@@ -899,24 +926,6 @@ class Dataset(object):
break
s += 1
def _get_context_window_left_right(self):
"""
:return: (ctx_left, ctx_right)
:rtype: None|(NumbersDict,NumbersDict)
"""
if self.context_window:
# One less because the original frame also counts, and context_window=1 means that we just have that single frame.
# ctx_total is how much frames we add additionally.
ctx_total = NumbersDict.max([self.context_window, 1]) - 1
# In case ctx_total is odd / context_window is even, we have to decide where to put one more frame.
# To keep it consistent with e.g. 1D convolution with a kernel of even size, we add one more to the right.
# See test_tfconv1d_evensize().
ctx_left = ctx_total // 2
ctx_right = ctx_total - ctx_left
return ctx_left, ctx_right
else:
return None
def get_start_end_frames_full_seq(self, seq_idx):
"""
:param int seq_idx:
......@@ -925,10 +934,8 @@ class Dataset(object):
"""
end = self.get_seq_length(seq_idx)
start = NumbersDict.constant_like(0, numbers_dict=end)
ctx_lr = self._get_context_window_left_right()
if ctx_lr:
start -= ctx_lr[0]
end += ctx_lr[1]
start -= self.ctx_left
end += self.ctx_right
return start, end
def sample(self, seq_idx):
......@@ -986,7 +993,6 @@ class Dataset(object):
print("Non-recurrent network, chunk size %s:%s ignored" % (chunk_size, chunk_step), file=log.v4)
chunk_size = 0
batch = Batch()
ctx_lr = self._get_context_window_left_right()
total_num_seqs = 0
last_seq_idx = -1
avg_weight = sum([v[0] for v in self.weights.values()]) / (len(self.weights.keys()) or 1)
......@@ -999,9 +1005,8 @@ class Dataset(object):
continue
if total_num_seqs > max_total_num_seqs:
break
if ctx_lr:
t_start -= ctx_lr[0]
t_end += ctx_lr[1]
t_start -= self.ctx_left
t_end += self.ctx_right
if recurrent_net:
length = t_end - t_start
if length.any_compare(max_seq_length, (lambda a, b: a > b)):
......@@ -1157,12 +1162,14 @@ def get_dataset_class(name):
def init_dataset(kwargs, extra_kwargs=None, default_kwargs=None):
"""
:param dict[str]|str|(()->dict[str]) kwargs:
:param dict[str]|str|(()->dict[str])|Dataset kwargs:
:param dict[str]|None extra_kwargs:
:param dict[str]|None default_kwargs:
:rtype: Dataset
"""
assert kwargs
if isinstance(kwargs, Dataset):
return kwargs
if callable(kwargs):
return init_dataset(kwargs(), extra_kwargs=extra_kwargs, default_kwargs=default_kwargs)
if isinstance(kwargs, (str, unicode)):
......
......@@ -18,8 +18,6 @@ class EngineBase(object):
Base class for a backend engine, such as :class:`TFEngine.Engine`.
"""
_epoch_model = None # type: typing.Optional[typing.Tuple[typing.Optional[int],typing.Optional[str]]] # get_epoch_model() # nopep8
def __init__(self):
self.epoch = 0
self.pretrain = None # type: typing.Optional[Pretrain]
......@@ -67,10 +65,6 @@ class EngineBase(object):
:returns (epoch, modelFilename)
:rtype: (int|None, str|None)
"""
# XXX: We cache it, although this is wrong if we have changed the config.
if cls._epoch_model:
return cls._epoch_model
start_epoch_mode = config.value('start_epoch', 'auto')
if start_epoch_mode == 'auto':
start_epoch = None
......@@ -134,7 +128,6 @@ class EngineBase(object):
print("warning: start_epoch %i but there is %s" % (start_epoch, epoch_model), file=log.v4)
epoch_model = start_epoch - 1, existing_models[start_epoch - 1]
cls._epoch_model = epoch_model
return epoch_model
@classmethod
......
......@@ -251,6 +251,8 @@ class HDFDataset(CachedDataset):
if key == "data":
inputs = fin['inputs']
data = inputs[start_pos[0]:end_pos[0]]
if self.window > 1:
data = self.sliding_window(data)
else:
assert 'targets' in fin
targets = fin['targets/data/' + key]
......@@ -1149,6 +1151,8 @@ class HDFDatasetWriter:
print("Data keys:", data_keys, file=log.v3)
if "orth" in data_keys: # special workaround for now, not handled
data_keys.remove("orth")
if "raw" in data_keys:
data_keys.remove("raw")
data_target_keys = [key for key in dataset.get_target_list() if key in data_keys]
data_input_keys = [key for key in data_keys if key not in data_target_keys]
assert len(data_input_keys) > 0 and len(data_target_keys) > 0
......
......@@ -238,9 +238,10 @@ class LearningRateControl(object):
relative_error = (new_error - old_error) / abs(new_error)
if self.relative_error_also_relative_to_learning_rate:
learning_rate = self.get_most_recent_learning_rate(new_epoch, exclude_current=False)
# If the learning rate is lower than the initial learning rate,
# the relative error is also expected to be lower, so correct for that here.
relative_error /= learning_rate / self.default_learning_rate
if learning_rate > 0:
# If the learning rate is lower than the initial learning rate,
# the relative error is also expected to be lower, so correct for that here.
relative_error /= learning_rate / self.default_learning_rate
return relative_error
def set_epoch_error(self, epoch, error):
......
......@@ -22,7 +22,6 @@ from ActivationFunctions import strtoact, strtoact_single_joined, elu
import TheanoUtil
from TheanoUtil import class_idx_seq_to_1_of_k
from Log import log
from cuda_implementation.FractionalMaxPoolingOp import fmp
from math import ceil
from theano.sandbox.rng_mrg import MRG_RandomStreams as RandomStreams
from TheanoUtil import print_to_file, DumpOp
......
......@@ -18,6 +18,7 @@ from threading import Event, Thread
import typing
import numpy
from Dataset import Dataset, init_dataset
from SprintDataset import SprintDatasetBase
from EngineBase import EngineBase
from Log import log
......@@ -27,6 +28,16 @@ from Util import get_gpu_names, interrupt_main, to_bool, BackendEngine
import TaskSystem
import rnn
if typing.TYPE_CHECKING:
try:
import TFEngine
except ImportError:
TFEngine = None
try:
import Engine as TheanoEngine
except ImportError:
TheanoEngine = None
_rnn_file = rnn.__file__
_main_file = getattr(sys.modules["__main__"], "__file__", "")
if _rnn_file.endswith(".pyc"):
......@@ -48,10 +59,11 @@ MaxSegmentLength = 1
TargetMode = None # type: typing.Optional[str]
Task = "train"
Engine = None # type: typing.Optional[typing.Union[typing.Type["TFEngine.Engine"],typing.Type["Engine.Engine"]]]
Engine = None # type: typing.Optional[typing.Union[typing.Type[TFEngine.Engine],typing.Type[TheanoEngine.Engine]]]
config = None # type: typing.Optional[rnn.Config]
sprintDataset = None # type: typing.Optional[SprintDatasetBase]
engine = None # type: Engine
customDataset = None # type: typing.Optional[Dataset]
engine = None # type: typing.Optional[typing.Union[TFEngine.Engine,TheanoEngine.Engine,Engine]]
# <editor-fold desc="generic init">
......@@ -814,13 +826,18 @@ def _prepare_forwarding():
def _init_dataset():
global sprintDataset
global sprintDataset, customDataset
if sprintDataset:
return
assert config
extra_opts = config.typed_value("sprint_interface_dataset_opts", {})
assert isinstance(extra_opts, dict)
sprintDataset = SprintDatasetBase.from_config(config, **extra_opts)
if config.is_true("sprint_interface_custom_dataset"):
custom_dataset_func = config.typed_value("sprint_interface_custom_dataset")
assert callable(custom_dataset_func)
custom_dataset_opts = custom_dataset_func(sprint_dataset=sprintDataset)
customDataset = init_dataset(custom_dataset_opts)
def get_final_epoch():
......@@ -890,12 +907,21 @@ def features_to_dataset(features, segment_name):
num_time = features.shape[1]
assert features.shape == (InputDim, num_time)
if customDataset:
customDataset.finish_epoch() # reset
customDataset.init_seq_order(epoch=1, seq_list=[segment_name])
# Fill the data for the current segment.
sprintDataset.shuffle_frames_of_nseqs = 0 # We must not shuffle.
sprintDataset.init_sprint_epoch(None) # Reset cache. We don't need old seqs anymore.
sprintDataset.init_seq_order()
sprintDataset.init_seq_order(epoch=1, seq_list=[segment_name])
seq = sprintDataset.add_new_data(features, segment_name=segment_name)
return sprintDataset, seq
if not customDataset:
return sprintDataset, seq
assert seq == 0
return customDataset, 0
def _forward(segment_name, features):
......
......@@ -59,7 +59,8 @@ class Runner(object):
:param bool train: whether to do updates on the model
:param bool|None train_flag: normally just as train. but e.g. maybe you want to have the train_flag but not train
:param bool eval: whether to evaluate (i.e. calculate loss/error)
:param dict[str,tf.Tensor|TFUtil.Data|TFNetworkLayer.LayerBase]|None extra_fetches: additional fetches per step.
:param dict[str,tf.Tensor|TFUtil.Data|TFNetworkLayer.LayerBase|()->tf.Tensor)]|None extra_fetches:
additional fetches per step.
`extra_fetches_callback` will be called with these. In case of Data/LayerBase, it will return a list,
where each item corresponds to the batch-seq.
It might also be useful to add `network.get_extern_data("seq_idx")` and `network.get_extern_data("seq_tag")`.
......@@ -108,13 +109,8 @@ class Runner(object):
:return: values and actions which should be calculated and executed in self.run() by the TF session for each step
:rtype: dict[str,tf.Tensor|tf.Operation]
"""
# Note that it is important that we do not recreate graph nodes for every call to this function.
# Thus everything which we access here should be cached.
d = self.engine.network.get_fetches_dict(
config=self.engine.config,
should_train=self._should_train, should_eval=self._should_eval,
with_summary=True, with_size=True)
d = {}
optim_op = None
if self._should_train:
assert self.engine.updater
......@@ -125,8 +121,19 @@ class Runner(object):
# Force a new check.
self.engine._checked_uninitialized_vars = False
self.engine.updater.init_optimizer_vars(session=self.engine.tf_session)
# This function has to be called before `get_fetches_dict(..., with_summary=True)`,
# because it may introduce new summaries to the graph which are later collected.
optim_op = self.engine.updater.get_optim_op(callback_on_new=callback_on_new)
d["optim_op"] = self.engine.updater.get_optim_op(callback_on_new=callback_on_new)
# Note that it is important that we do not recreate graph nodes for every call to this function.
# Thus everything which we access here should be cached.
d_fetches = self.engine.network.get_fetches_dict(
config=self.engine.config,
should_train=self._should_train, should_eval=self._should_eval,
with_summary=True, with_size=True)
d.update(d_fetches)
if optim_op is not None:
d["optim_op"] = optim_op
if self.engine.updater.optim_meta_losses_dict:
d.update(self.engine.updater.optim_meta_losses_dict)
......@@ -141,6 +148,11 @@ class Runner(object):
continue
if isinstance(v, LayerBase):
v = v.output
if callable(v):
v = v()
assert isinstance(v, tf.Tensor)
d["extra:%s" % k] = v
continue
assert isinstance(v, Data)
d["extra:%s" % k] = v.placeholder # see _maybe_handle_extra_fetches, it will transform to batch-major there
for i, s in v.size_placeholder.items():
......@@ -345,6 +357,9 @@ class Runner(object):
continue
if isinstance(v, LayerBase):
v = v.output
if callable(v):
d[k] = fetches_results["extra:%s" % k]
continue
assert isinstance(v, Data)
if v.batch_dim_axis != 0:
r = numpy.moveaxis(r, v.batch_dim_axis, 0)
......@@ -581,6 +596,14 @@ class Runner(object):
elapsed_time_tf += self._horovod_sync_params(local_step=step)
duration = time.time() - start_time
self._print_process(report_prefix=report_prefix, step=step, step_duration=duration, eval_info=eval_info)
if self.engine.config.bool("stop_on_nonfinite_train_score", True):
score_values = self._results_accumulated.values()
if any(numpy.isinf(score_values)) or any(numpy.isnan(score_values)):
print("Model seems broken, got inf or nan score.", file=log.v1)
print("Accumulated scores:", self._results_accumulated, file=log.v1)
raise Exception("Inf/nan score in step %i." % step)
step += 1
if self.cancel_flag:
raise CancelTrainingException("cancel_flag is set")
......@@ -615,6 +638,7 @@ class Runner(object):
print("Exception %r in step %r." % (exc, step), file=log.v1)
if not isinstance(exc, CancelTrainingException):
help_on_tf_exception(
session=sess,
exception=exc, fetches=fetches_dict, feed_dict=feed_dict, meta_step_info=meta_step_info,
extern_data=self.data_provider.extern_data, file=log.v2)
sys.excepthook(*sys.exc_info())
......@@ -1916,6 +1940,7 @@ class Engine(EngineBase):
:param str output_file:
:param str output_file_format: "txt" or "py"
"""
from TFNetworkLayer import LayerBase
print("Search with network on %r." % dataset, file=log.v1)
if not self.use_search_flag or not self.network or self.use_dynamic_train_flag:
self.use_search_flag = True
......@@ -1956,22 +1981,22 @@ class Engine(EngineBase):
num_targets = len(output_layer_names)
# Create lists with information about the output layers. All of length num_targets.
output_layers = [] # list[LayerBase]
out_beam_sizes = [] # list[int|None]
output_layer_beam_scores = [] # list[tf.Tensor|None]
target_keys = [] # list[str]
output_layers = [] # type: typing.List[LayerBase]
out_beam_sizes = [] # type: typing.List[typing.Optional[int]]
output_layer_beam_scores = [] # type: typing.List[typing.Optional[tf.Tensor]]
target_keys = [] # type: typing.List[str]
for output_layer_name in output_layer_names:
output_layer = self.network.layers[output_layer_name]
output_layers.append(output_layer)
out_beam_size = output_layer.output.beam_size
if out_beam_size is None:
out_beam = output_layer.output.beam
if out_beam is None:
print("Given output %r is after decision (no beam)." % output_layer, file=log.v1)
output_layer_beam_scores.append(None)
else:
print("Given output %r has beam size %i." % (output_layer, out_beam_size), file=log.v1)
print("Given output %r has beam %s." % (output_layer, out_beam), file=log.v1)
output_layer_beam_scores.append(output_layer.get_search_choices().beam_scores)
out_beam_sizes.append(out_beam_size)
out_beam_sizes.append(out_beam.beam_size if out_beam else None)
target_keys.append(output_layer.target or self.network.extern_data.default_target)
out_cache = None
......@@ -2133,7 +2158,7 @@ class Engine(EngineBase):
output_layer = self.network.layers[output_layer_name]
output_t = output_layer.output.get_placeholder_as_batch_major()
output_seq_lens_t = output_layer.output.get_sequence_lengths()
out_beam_size = output_layer.output.beam_size
out_beam_size = output_layer.output.beam.beam_size
output_layer_beam_scores_t = None
if out_beam_size is None:
print("Given output %r is after decision (no beam)." % output_layer, file=log.v4)
......@@ -2330,7 +2355,7 @@ class Engine(EngineBase):
output_layer = self.network.layers[output_layer_name]
output_t = output_layer.output.get_placeholder_as_batch_major()
output_seq_lens_t = output_layer.output.get_sequence_lengths()
out_beam_size = output_layer.output.beam_size
out_beam_size = output_layer.output.beam.beam_size
output_layer_beam_scores_t = None
if out_beam_size is None:
print("Given output %r is after decision (no beam)." % output_layer, file=log.v1)
......
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
......@@ -403,8 +403,9 @@ class MultiChannelMultiResolutionStftLayer(_ConcatInputLayer):
return input_placeholder.shape[2]
def _apply_stft_to_input(self):
from TFUtil import get_shape
def _cropStftOutputToReferenceFrameSizeLength(channel_concatenated_stft, crop_size):
return tf.slice(channel_concatenated_stft, [0, 0, 0], [tf.shape(channel_concatenated_stft)[0], crop_size, tf.shape(channel_concatenated_stft)[2]])
return tf.slice(channel_concatenated_stft, [0, 0, 0], [get_shape(channel_concatenated_stft)[0], crop_size, get_shape(channel_concatenated_stft)[2]])
input_placeholder = self.input_data.get_placeholder_as_batch_major()
channel_wise_stft_res_list = list()
......@@ -421,7 +422,7 @@ class MultiChannelMultiResolutionStftLayer(_ConcatInputLayer):
return window
def _padTimeSignal(input_placeholder, frame_size):
if frame_size > self._reference_frame_size:
return tf.concat([input_signal, tf.ones([tf.shape(input_signal)[0], frame_size-self._reference_frame_size, tf.shape(input_signal)[2]])*1e-7], axis=1)
return tf.concat([input_signal, tf.ones([get_shape(input_signal)[0], frame_size-self._reference_frame_size, get_shape(input_signal)[2]])*1e-7], axis=1)
else:
return input_placeholder
......@@ -441,7 +442,7 @@ class MultiChannelMultiResolutionStftLayer(_ConcatInputLayer):
concat_feature_dim = channel_wise_stft.shape[2] * channel_wise_stft.shape[3]
channel_concatenated_stft = tf.reshape(channel_wise_stft, (batch_dim, time_dim, concat_feature_dim))
if channel_wise_stft_res_list:
channel_concatenated_stft = _cropStftOutputToReferenceFrameSizeLength(channel_concatenated_stft, tf.shape(channel_wise_stft_res_list[0])[1])
channel_concatenated_stft = _cropStftOutputToReferenceFrameSizeLength(channel_concatenated_stft, get_shape(channel_wise_stft_res_list[0])[1])
channel_wise_stft_res_list.append(channel_concatenated_stft)
output_placeholder = tf.concat(channel_wise_stft_res_list, axis=2)
return output_placeholder
......
......@@ -327,7 +327,8 @@ class Updater(object):
self.optim_op = tf.group(self.optim_op, add_check_numerics_ops([self.optim_op]))
# Do this at the very end.
incr_step_op = tf.assign_add(self.network.global_train_step, 1, name="global_train_step_increment")
with tf.control_dependencies([self.optim_op]):
incr_step_op = tf.assign_add(self.network.global_train_step, 1, name="global_train_step_increment")
self.optim_op = tf.group(self.optim_op, incr_step_op, name="optim_and_step_incr")
if self.config.bool("debug_save_updater_vars", False):
......
This diff is collapsed.
......@@ -1505,16 +1505,18 @@ def json_remove_comments(string, strip_space=True):
def _py2_unicode_to_str_recursive(s):
"""
This is supposed to be run with Python 2.
Also see :func:`as_str` and :func:`py2_utf8_str_to_unicode`.
:param str|unicode s:
:param str|unicode s: or any recursive structure such as dict, list, tuple
:return: Python 2 str (is like Python 3 UTF-8 formatted bytes)
:rtype: str
"""
if isinstance(s, dict):
return {_py2_unicode_to_str_recursive(key): _py2_unicode_to_str_recursive(value) for key, value in s.items()}
elif isinstance(s, list):
return [_py2_unicode_to_str_recursive(element) for element in s]
elif isinstance(s, (list, tuple)):
return make_seq_of_type(type(s), [_py2_unicode_to_str_recursive(element) for element in s])
elif isinstance(s, unicode):
return s.encode('utf-8')
return s.encode('utf-8') # Python 2 str, Python 3 bytes
else:
return s
......@@ -2107,7 +2109,7 @@ def to_bool(v):
def as_str(s):
"""
:param str|unicode|bytes s:
:rtype: str
:rtype: str|unicode
"""
if isinstance(s, str) or 'unicode' in str(type(s)):
return s
......@@ -2123,6 +2125,7 @@ def py2_utf8_str_to_unicode(s):
but just using "äöü" will actually be the raw utf8 byte sequence.
This can happen when you eval() some string.
We assume that you are using Python 2, and got the string (not unicode object) "äöü", or maybe "abc".
Also see :func:`_py2_unicode_to_str_recursive` and :func:`as_str`.
:return: if it is indeed unicode, it will return the unicode object, otherwise it keeps the string
:rtype: str|unicode
"""
......@@ -2138,6 +2141,31 @@ def py2_utf8_str_to_unicode(s):
return s.decode("utf8")
def unicode_to_str(s):
"""
The behavior is different depending on Python 2 or Python 3. In all cases, the returned type is a str object.
Python 2:
We return the utf8 encoded str (which is like Python 3 bytes, or for ASCII, there is no difference).
Python 3:
We return a str object.
Note that this function probably does not make much sense.
It might be used when there is other code which expects a str object, no matter if Python 2 or Python 3.
In Python 2, a str object often holds UTF8 text, so the behavior of this function is fine then.
Also see :func:`as_str`.
:param str|unicode|bytes s:
:rtype: str
"""
if PY3 and isinstance(s, bytes):
s = s.decode("utf8")
assert isinstance(s, str)
if not PY3 and isinstance(s, unicode):
s = s.encode("utf8")
assert isinstance(s, str)
assert isinstance(s, str)
return s
def deepcopy(x):
"""
Simpler variant of copy.deepcopy().
......
......@@ -30,7 +30,7 @@ network = {
"c": {"class": "eval", "from": ["input_gate", "cell_in", "forget_gate", "prev:c"],
"eval": "source(0) * source(1) + source(2) * source(3)"},
"output": {"class": "eval", "from": ["output_gate", "c"],
"eval": "source(0) * source(1)"},
"eval": "source(0) * source(1)", "out_type": {"shape": (10,)}},
}},
"output": {"class": "softmax", "loss": "ce", "from": "lstm"}
}
......
......@@ -7,10 +7,20 @@ Debugging
debug_add_check_numerics_on_output
If set to ``True``, should assert for ``inf`` and ``nan``.
debug_grad_summaries
If set to ``True``, adds additional information about the gradients to the TensorBoard.
debug_print_layer_output_template
If set to ``True``, print the layer template information during network construction.
debug_print_layer_output_shape
If set to ``True``, print the layer shape information while the graph is executed.
debug_objective_loss_summaries
If set to ``True``, adds the objective loss values (normalization only when activated, including loss scaling)
to the TensorBoard.
debug_unnormalized_loss_summaries
If set to ``True``, adds the unnormalized loss values to the TensorBoard
Also see :ref:`debugging`.
......@@ -14,6 +14,10 @@ forward_override_hdf_output
Per default, Returnn will give an error when trying to overwrite an existing output. If this flag is set to true,
the check is disabled.
output_file
When the task is "forward", specifies the output path for the resulting hdf. If not specified,
the name will be "dump-fwd-epoch-%i.hdf" % epoch.
search_output_layer
TODO...
......
......@@ -36,11 +36,11 @@ if typing.TYPE_CHECKING:
try:
import TFEngine
except ImportError:
pass
TFEngine = None
try:
import Engine
except ImportError:
pass
Engine = None
config = None # type: typing.Optional[Config]
engine = None # type: typing.Optional[typing.Union[TFEngine.Engine,Engine.Engine]]
......
......@@ -136,9 +136,10 @@ def get_test_tmp_file(suffix=".hdf"):
_hdf_cache = {} # opts -> hdf fn
def generate_hdf_from_other(opts):
def generate_hdf_from_other(opts, suffix=".hdf"):
"""
:param dict[str] opts:
:param str suffix:
:return: hdf filename
:rtype: str
"""
......@@ -147,7 +148,7 @@ def generate_hdf_from_other(opts):
cache_key = make_hashable(opts)
if cache_key in _hdf_cache:
return _hdf_cache[cache_key]
fn = get_test_tmp_file(suffix=".hdf")
fn = get_test_tmp_file(suffix=suffix)
from Dataset import init_dataset
dataset = init_dataset(opts)
hdf_dataset = HDFDatasetWriter(fn)
......
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
#!/usr/bin/env python3
"""
Construct/compile the computation graph, and optionally save it to some file.
There are various options/variations for what task and what conditions you can create the graph,
e.g. for training, forwarding, search, or also step-by-step execution over a recurrent layer.
"""
from __future__ import print_function
......@@ -15,6 +20,7 @@ sys.path.insert(0, returnn_dir)
import rnn
from Log import log
from Config import Config
import argparse
import Util
from Util import NotSpecified
......@@ -24,6 +30,9 @@ from TFNetworkLayer import LayerBase, register_layer_class, WrappedInternalLayer
from TFNetworkRecLayer import RecLayer, _SubnetworkRecCell, ChoiceLayer
config = None # type: typing.Optional[Config]
def init(config_filename, log_verbosity):
"""
:param str config_filename: filename to config-file
......@@ -42,7 +51,7 @@ def init(config_filename, log_verbosity):
global config
config = rnn.config
rnn.init_log()
print("Returnn compile-native-op starting up.", file=log.v1)
print("Returnn compile-tf-graph starting up.", file=log.v1)
rnn.returnn_greeting()
rnn.init_backend_engine()
assert Util.BackendEngine.is_tensorflow_selected(), "this is only for TensorFlow"
......@@ -184,6 +193,10 @@ class RecStepByStepLayer(RecLayer):
print("Stored rec-step-by-step info JSON in file:", output_file_name)
class StateVar:
"""
Represents a state variable, i.e. either a state, a choice, or encoder state, etc.
"""
def __init__(self, name, initial_value, data_shape):
"""
:param str name:
......@@ -509,12 +522,14 @@ class ChoiceStateVarLayer(LayerBase):
d["explicit_search_sources"] = [get_layer(name) for name in d["explicit_search_sources"]]
super(ChoiceStateVarLayer, cls).transform_config_dict(d, network=network, get_layer=get_layer)
@classmethod
def get_out_data_from_opts(cls, **kwargs):
return ChoiceLayer.get_out_data_from_opts(**kwargs)
get_out_data_from_opts = ChoiceLayer.get_out_data_from_opts
class SubnetworkRecCellSingleStep(_SubnetworkRecCell):
"""
Adapts :class:`_SubnetworkRecCell` such that we execute only a single step.
"""
def __init__(self, **kwargs):
self._parent_layers = {} # type: typing.Dict[str,WrappedInternalLayer]
super(SubnetworkRecCellSingleStep, self).__init__(**kwargs)
......@@ -534,10 +549,16 @@ class SubnetworkRecCellSingleStep(_SubnetworkRecCell):
output = layer.output.copy()
output.placeholder = rec_layer.create_state_var(
name="base_value_%s" % layer_name, initial_value=output.placeholder, data_shape=output)
output.size_placeholder = {
i: rec_layer.create_state_var(
name="base_size%i_%s" % (i, layer_name), initial_value=size)
for (i, size) in output.size_placeholder.items()}
from TFUtil import DimensionTag
for i, size in list(output.size_placeholder.items()):
dim_tag = DimensionTag.get_tag_from_size_tensor(size)
if not dim_tag:
print("Warning, no defined dim tag on %r, axis %i" % (layer, output.get_batch_axis(i)), file=log.v2)
dim_tag = output.get_dim_tag(output.get_batch_axis(i))
dim_tag.set_tag_on_size_tensor(size)
new_size = rec_layer.create_state_var(name="base_size%i_%s" % (i, layer_name), initial_value=size)
dim_tag.set_tag_on_size_tensor(new_size)
output.size_placeholder[i] = new_size
layer = WrappedInternalLayer(name=layer_name, network=self.parent_net, output=output, base_layer=layer)
self._parent_layers[layer_name] = layer
return layer
......@@ -616,7 +637,7 @@ def main(argv):
argparser.add_argument("--output_file_model_params_list", help="line-based, names of model params")
argparser.add_argument("--output_file_state_vars_list", help="line-based, name of state vars")
args = argparser.parse_args(argv[1:])
assert args.train in [0, 1, 2] and args.eval in [0, 1] and args.search in [0, 1]
assert args.train in [0, 1, -1] and args.eval in [0, 1] and args.search in [0, 1]
init(config_filename=args.config, log_verbosity=args.verbosity)
assert 'network' in config.typed_dict
net_dict = config.typed_dict["network"]
......