Python tensor2tensor.models.transformer.transformer_prepare_encoder() Examples
The following are 30
code examples of tensor2tensor.models.transformer.transformer_prepare_encoder().
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example.
You may also want to check out all available functions/classes of the module
tensor2tensor.models.transformer
, or try the search function
.
.
Example #1
| Source File: transformer_moe.py From BERT with Apache License 2.0 | 6 votes |
|
def _prepare_encoder(self, inputs, target_space):
"""Process the transformer encoder inputs."""
inputs = common_layers.flatten4d3d(inputs)
output = transformer.transformer_prepare_encoder(
inputs,
target_space,
self._hparams,
features=None,
)
enco_input, enco_self_att_bias, enco_deco_att_bias = output
enco_input = tf.nn.dropout(
enco_input, 1.0 - self._hparams.layer_prepostprocess_dropout)
return enco_input, enco_self_att_bias, enco_deco_att_bias
Example #2
| Source File: similarity_transformer.py From BERT with Apache License 2.0 | 6 votes |
|
def encode(self, features, input_key):
hparams = self._hparams
inputs = common_layers.flatten4d3d(features[input_key])
(encoder_input, encoder_self_attention_bias, _) = (
transformer.transformer_prepare_encoder(inputs, problem.SpaceID.EN_TOK,
hparams))
encoder_input = tf.nn.dropout(encoder_input,
1.0 - hparams.layer_prepostprocess_dropout)
encoder_output = transformer.transformer_encoder(
encoder_input,
encoder_self_attention_bias,
hparams,
nonpadding=transformer.features_to_nonpadding(features, input_key))
encoder_output = tf.reduce_mean(encoder_output, axis=1)
return encoder_output
Example #3
| Source File: transformer_moe.py From fine-lm with MIT License | 6 votes |
|
def _prepare_encoder(self, inputs, target_space):
"""Process the transformer encoder inputs."""
inputs = common_layers.flatten4d3d(inputs)
output = transformer.transformer_prepare_encoder(
inputs,
target_space,
self._hparams,
features=None,
)
enco_input, enco_self_att_bias, enco_deco_att_bias = output
enco_input = tf.nn.dropout(
enco_input, 1.0 - self._hparams.layer_prepostprocess_dropout)
return enco_input, enco_self_att_bias, enco_deco_att_bias
Example #4
| Source File: similarity_transformer.py From training_results_v0.5 with Apache License 2.0 | 6 votes |
|
def encode(self, features, input_key):
hparams = self._hparams
inputs = common_layers.flatten4d3d(features[input_key])
(encoder_input, encoder_self_attention_bias, _) = (
transformer.transformer_prepare_encoder(inputs, problem.SpaceID.EN_TOK,
hparams))
encoder_input = tf.nn.dropout(encoder_input,
1.0 - hparams.layer_prepostprocess_dropout)
encoder_output = transformer.transformer_encoder(
encoder_input,
encoder_self_attention_bias,
hparams,
nonpadding=transformer.features_to_nonpadding(features, input_key))
encoder_output = tf.reduce_mean(encoder_output, axis=1)
return encoder_output
Example #5
| Source File: latent_layers.py From fine-lm with MIT License | 6 votes |
|
def transformer_text_encoder(x,
space_id,
hparams,
name="transformer_text_encoder"):
"""Transformer text encoder over inputs with unmasked full attention.
Args:
x: Tensor of shape [batch, length, 1, hparams.hidden_size].
space_id: int, id.
hparams: tf.contrib.training.HParams.
name: string, variable scope.
Returns:
encoder_output: Tensor of shape [batch, length, hparams.hidden_size].
ed: Tensor of shape [batch, 1, 1, length]. Encoder-decoder attention bias
for any padded tokens.
"""
with tf.variable_scope(name):
x = common_layers.flatten4d3d(x)
(encoder_input, encoder_self_attention_bias,
ed) = transformer.transformer_prepare_encoder(x, space_id, hparams)
encoder_input = tf.nn.dropout(encoder_input, 1.0 - hparams.dropout)
encoder_output = transformer.transformer_encoder(
encoder_input, encoder_self_attention_bias, hparams)
return encoder_output, ed
Example #6
| Source File: transformer_moe.py From training_results_v0.5 with Apache License 2.0 | 6 votes |
|
def _prepare_encoder(self, inputs, target_space):
"""Process the transformer encoder inputs."""
inputs = common_layers.flatten4d3d(inputs)
output = transformer.transformer_prepare_encoder(
inputs,
target_space,
self._hparams,
features=None,
)
enco_input, enco_self_att_bias, enco_deco_att_bias = output
enco_input = tf.nn.dropout(
enco_input, 1.0 - self._hparams.layer_prepostprocess_dropout)
return enco_input, enco_self_att_bias, enco_deco_att_bias
Example #7
| Source File: universal_transformer_utils.py From BERT with Apache License 2.0 | 6 votes |
|
def universal_transformer_encoder(inputs, target_space,
hparams, features=None, make_image_summary=False):
encoder_input, self_attention_bias, encoder_decoder_attention_bias = (
transformer.transformer_prepare_encoder(
inputs, target_space, hparams, features=features))
encoder_input = tf.nn.dropout(encoder_input,
1.0 - hparams.layer_prepostprocess_dropout)
[encoder_output,
encoder_extra_output] = universal_transformer_util.universal_transformer_encoder(
encoder_input,
self_attention_bias,
hparams,
nonpadding=transformer.features_to_nonpadding(features, "inputs"),
save_weights_to=None,
make_image_summary=make_image_summary)
# encoder_output = tf.expand_dims(encoder_output, 2)
return encoder_output
Example #8
| Source File: transformer_moe.py From tensor2tensor with Apache License 2.0 | 6 votes |
|
def _prepare_encoder(self, inputs, target_space):
"""Process the transformer encoder inputs."""
inputs = common_layers.flatten4d3d(inputs)
output = transformer.transformer_prepare_encoder(
inputs,
target_space,
self._hparams,
features=None,
)
enco_input, enco_self_att_bias, enco_deco_att_bias = output
enco_input = tf.nn.dropout(
enco_input, 1.0 - self._hparams.layer_prepostprocess_dropout)
return enco_input, enco_self_att_bias, enco_deco_att_bias
Example #9
| Source File: similarity_transformer.py From tensor2tensor with Apache License 2.0 | 6 votes |
|
def encode(self, features, input_key):
hparams = self._hparams
inputs = common_layers.flatten4d3d(features[input_key])
(encoder_input, encoder_self_attention_bias, _) = (
transformer.transformer_prepare_encoder(inputs, problem.SpaceID.EN_TOK,
hparams))
encoder_input = tf.nn.dropout(encoder_input,
1.0 - hparams.layer_prepostprocess_dropout)
encoder_output = transformer.transformer_encoder(
encoder_input,
encoder_self_attention_bias,
hparams,
nonpadding=transformer.features_to_nonpadding(features, input_key))
encoder_output = tf.reduce_mean(encoder_output, axis=1)
return encoder_output
Example #10
| Source File: base_transformer_utils.py From BERT with Apache License 2.0 | 6 votes |
|
def transformer_encoder(inputs, target_space, hparams, features=None, losses=None):
encoder_input, self_attention_bias, encoder_decoder_attention_bias = (
transformer.transformer_prepare_encoder(
inputs, target_space, hparams, features=features))
encoder_input = tf.nn.dropout(encoder_input,
1.0 - hparams.layer_prepostprocess_dropout)
encoder_output = transformer.transformer_encoder(
encoder_input,
self_attention_bias,
hparams,
nonpadding=transformer.features_to_nonpadding(features, "inputs"),
save_weights_to=None,
losses=losses)
# encoder_output = tf.expand_dims(encoder_output, 2)
return encoder_output
Example #11
| Source File: transformer_vae.py From training_results_v0.5 with Apache License 2.0 | 5 votes |
|
def encode(x, x_space, hparams, name):
"""Transformer preparations and encoder."""
with tf.variable_scope(name):
(encoder_input, encoder_self_attention_bias,
ed) = transformer.transformer_prepare_encoder(x, x_space, hparams)
encoder_input = tf.nn.dropout(encoder_input, 1.0 - hparams.dropout)
return transformer.transformer_encoder(
encoder_input, encoder_self_attention_bias, hparams), ed
Example #12
| Source File: universal_transformer.py From BERT with Apache License 2.0 | 5 votes |
|
def encode(self, inputs, target_space, hparams, features=None, losses=None):
"""Encode transformer inputs.
Args:
inputs: Transformer inputs [batch_size, input_length, input_height,
hidden_dim] which will be flattened along the two spatial dimensions.
target_space: scalar, target space ID.
hparams: hyperparmeters for model.
features: optionally pass the entire features dictionary as well.
This is needed now for "packed" datasets.
losses: Unused.
Returns:
Tuple of:
encoder_output: Encoder representation.
[batch_size, input_length, hidden_dim]
encoder_extra_output: which is extra encoder output used in some
variants of the model (e.g. in ACT, to pass the ponder-time to body)
"""
del losses
inputs = common_layers.flatten4d3d(inputs)
(encoder_input, self_attention_bias, _) = (
transformer.transformer_prepare_encoder(inputs, target_space, hparams))
encoder_input = tf.nn.dropout(encoder_input,
1.0 - hparams.layer_prepostprocess_dropout)
(encoder_output, encoder_extra_output) = (
universal_transformer_util.universal_transformer_encoder(
encoder_input,
self_attention_bias,
hparams,
nonpadding=transformer.features_to_nonpadding(features, "inputs"),
save_weights_to=self.attention_weights))
return encoder_output, encoder_extra_output
Example #13
| Source File: transformer_revnet.py From training_results_v0.5 with Apache License 2.0 | 5 votes |
|
def body(self, features):
hparams = self._hparams
targets = features["targets"]
inputs = features["inputs"]
target_space = features["target_space_id"]
inputs = common_layers.flatten4d3d(inputs)
targets = common_layers.flatten4d3d(targets)
(encoder_input, encoder_self_attention_bias,
encoder_decoder_attention_bias) = (transformer.transformer_prepare_encoder(
inputs, target_space, hparams))
(decoder_input,
decoder_self_attention_bias) = transformer.transformer_prepare_decoder(
targets, hparams)
encoder_input = tf.nn.dropout(encoder_input,
1.0 - hparams.layer_prepostprocess_dropout)
decoder_input = tf.nn.dropout(decoder_input,
1.0 - hparams.layer_prepostprocess_dropout)
encoder_output = transformer_revnet_encoder(
encoder_input, encoder_self_attention_bias, hparams)
decoder_output = transformer_revnet_decoder(
decoder_input, encoder_output, decoder_self_attention_bias,
encoder_decoder_attention_bias, hparams)
decoder_output = tf.expand_dims(decoder_output, 2)
return decoder_output
Example #14
| Source File: transformer_nat.py From BERT with Apache License 2.0 | 5 votes |
|
def encode(x, x_space, hparams, name):
"""Transformer preparations and encoder."""
with tf.variable_scope(name):
(encoder_input, encoder_self_attention_bias,
ed) = transformer.transformer_prepare_encoder(x, x_space, hparams)
encoder_input = tf.nn.dropout(encoder_input, 1.0 - hparams.dropout)
return transformer.transformer_encoder(
encoder_input, encoder_self_attention_bias, hparams), ed
Example #15
| Source File: transformer_nat.py From training_results_v0.5 with Apache License 2.0 | 5 votes |
|
def encode(x, x_space, hparams, name):
"""Transformer preparations and encoder."""
with tf.variable_scope(name):
(encoder_input, encoder_self_attention_bias,
ed) = transformer.transformer_prepare_encoder(x, x_space, hparams)
encoder_input = tf.nn.dropout(encoder_input, 1.0 - hparams.dropout)
return transformer.transformer_encoder(
encoder_input, encoder_self_attention_bias, hparams), ed
Example #16
| Source File: universal_transformer.py From training_results_v0.5 with Apache License 2.0 | 5 votes |
|
def encode(self, inputs, target_space, hparams, features=None, losses=None):
"""Encode transformer inputs.
Args:
inputs: Transformer inputs [batch_size, input_length, input_height,
hidden_dim] which will be flattened along the two spatial dimensions.
target_space: scalar, target space ID.
hparams: hyperparmeters for model.
features: optionally pass the entire features dictionary as well.
This is needed now for "packed" datasets.
losses: Unused.
Returns:
Tuple of:
encoder_output: Encoder representation.
[batch_size, input_length, hidden_dim]
encoder_extra_output: which is extra encoder output used in some
variants of the model (e.g. in ACT, to pass the ponder-time to body)
"""
del losses
inputs = common_layers.flatten4d3d(inputs)
(encoder_input, self_attention_bias, _) = (
transformer.transformer_prepare_encoder(inputs, target_space, hparams))
encoder_input = tf.nn.dropout(encoder_input,
1.0 - hparams.layer_prepostprocess_dropout)
(encoder_output, encoder_extra_output) = (
universal_transformer_util.universal_transformer_encoder(
encoder_input,
self_attention_bias,
hparams,
nonpadding=transformer.features_to_nonpadding(features, "inputs"),
save_weights_to=self.attention_weights))
return encoder_output, encoder_extra_output
Example #17
| Source File: universal_transformer_modified.py From Graph-Transformer with Apache License 2.0 | 5 votes |
|
def encode(self, inputs, target_space, hparams, features=None, losses=None):
"""Encode transformer inputs.
Args:
inputs: Transformer inputs [batch_size, input_length, input_height,
hidden_dim] which will be flattened along the two spatial dimensions.
target_space: scalar, target space ID.
hparams: hyperparmeters for model.
features: optionally pass the entire features dictionary as well.
This is needed now for "packed" datasets.
losses: Unused.
Returns:
Tuple of:
encoder_output: Encoder representation.
[batch_size, input_length, hidden_dim]
encoder_extra_output: which is extra encoder output used in some
variants of the model (e.g. in ACT, to pass the ponder-time to body)
"""
del losses
inputs = common_layers.flatten4d3d(inputs)
(encoder_input, self_attention_bias, _) = (
transformer.transformer_prepare_encoder(inputs, target_space, hparams))
encoder_input = tf.nn.dropout(encoder_input,
1.0 - hparams.layer_prepostprocess_dropout)
(encoder_output, encoder_extra_output) = (
universal_transformer_util.universal_transformer_encoder(
encoder_input,
self_attention_bias,
hparams,
nonpadding=transformer.features_to_nonpadding(features, "inputs"),
save_weights_to=self.attention_weights))
return encoder_output, encoder_extra_output
Example #18
| Source File: transformer_revnet.py From fine-lm with MIT License | 5 votes |
|
def body(self, features):
hparams = self._hparams
targets = features["targets"]
inputs = features["inputs"]
target_space = features["target_space_id"]
inputs = common_layers.flatten4d3d(inputs)
targets = common_layers.flatten4d3d(targets)
(encoder_input, encoder_self_attention_bias,
encoder_decoder_attention_bias) = (transformer.transformer_prepare_encoder(
inputs, target_space, hparams))
(decoder_input,
decoder_self_attention_bias) = transformer.transformer_prepare_decoder(
targets, hparams)
encoder_input = tf.nn.dropout(encoder_input,
1.0 - hparams.layer_prepostprocess_dropout)
decoder_input = tf.nn.dropout(decoder_input,
1.0 - hparams.layer_prepostprocess_dropout)
encoder_output = transformer_revnet_encoder(
encoder_input, encoder_self_attention_bias, hparams)
decoder_output = transformer_revnet_decoder(
decoder_input, encoder_output, decoder_self_attention_bias,
encoder_decoder_attention_bias, hparams)
decoder_output = tf.expand_dims(decoder_output, 2)
return decoder_output
Example #19
| Source File: transformer_vae.py From BERT with Apache License 2.0 | 5 votes |
|
def encode(x, x_space, hparams, name):
"""Transformer preparations and encoder."""
with tf.variable_scope(name):
(encoder_input, encoder_self_attention_bias,
ed) = transformer.transformer_prepare_encoder(x, x_space, hparams)
encoder_input = tf.nn.dropout(encoder_input, 1.0 - hparams.dropout)
return transformer.transformer_encoder(
encoder_input, encoder_self_attention_bias, hparams), ed
Example #20
| Source File: transformer_revnet.py From BERT with Apache License 2.0 | 5 votes |
|
def body(self, features):
hparams = self._hparams
targets = features["targets"]
inputs = features["inputs"]
target_space = features["target_space_id"]
inputs = common_layers.flatten4d3d(inputs)
targets = common_layers.flatten4d3d(targets)
(encoder_input, encoder_self_attention_bias,
encoder_decoder_attention_bias) = (transformer.transformer_prepare_encoder(
inputs, target_space, hparams))
(decoder_input,
decoder_self_attention_bias) = transformer.transformer_prepare_decoder(
targets, hparams)
encoder_input = tf.nn.dropout(encoder_input,
1.0 - hparams.layer_prepostprocess_dropout)
decoder_input = tf.nn.dropout(decoder_input,
1.0 - hparams.layer_prepostprocess_dropout)
encoder_output = transformer_revnet_encoder(
encoder_input, encoder_self_attention_bias, hparams)
decoder_output = transformer_revnet_decoder(
decoder_input, encoder_output, decoder_self_attention_bias,
encoder_decoder_attention_bias, hparams)
decoder_output = tf.expand_dims(decoder_output, 2)
return decoder_output
Example #21
| Source File: universal_transformer.py From tensor2tensor with Apache License 2.0 | 5 votes |
|
def encode(self, inputs, target_space, hparams, features=None, losses=None):
"""Encode transformer inputs.
Args:
inputs: Transformer inputs [batch_size, input_length, input_height,
hidden_dim] which will be flattened along the two spatial dimensions.
target_space: scalar, target space ID.
hparams: hyperparmeters for model.
features: optionally pass the entire features dictionary as well.
This is needed now for "packed" datasets.
losses: Unused.
Returns:
Tuple of:
encoder_output: Encoder representation.
[batch_size, input_length, hidden_dim]
encoder_extra_output: which is extra encoder output used in some
variants of the model (e.g. in ACT, to pass the ponder-time to body)
"""
del losses
inputs = common_layers.flatten4d3d(inputs)
(encoder_input, self_attention_bias, _) = (
transformer.transformer_prepare_encoder(inputs, target_space, hparams))
encoder_input = tf.nn.dropout(encoder_input,
1.0 - hparams.layer_prepostprocess_dropout)
(encoder_output, encoder_extra_output) = (
universal_transformer_util.universal_transformer_encoder(
encoder_input,
self_attention_bias,
hparams,
nonpadding=transformer.features_to_nonpadding(features, "inputs"),
save_weights_to=self.attention_weights))
return encoder_output, encoder_extra_output
Example #22
| Source File: transformer_nat.py From tensor2tensor with Apache License 2.0 | 5 votes |
|
def encode(x, x_space, hparams, name):
"""Transformer preparations and encoder."""
with tf.variable_scope(name):
(encoder_input, encoder_self_attention_bias,
ed) = transformer.transformer_prepare_encoder(x, x_space, hparams)
encoder_input = tf.nn.dropout(encoder_input, 1.0 - hparams.dropout)
return transformer.transformer_encoder(
encoder_input, encoder_self_attention_bias, hparams), ed
Example #23
| Source File: transformer_vae.py From tensor2tensor with Apache License 2.0 | 5 votes |
|
def encode(x, x_space, hparams, name):
"""Transformer preparations and encoder."""
with tf.variable_scope(name):
(encoder_input, encoder_self_attention_bias,
ed) = transformer.transformer_prepare_encoder(x, x_space, hparams)
encoder_input = tf.nn.dropout(encoder_input, 1.0 - hparams.dropout)
return transformer.transformer_encoder(
encoder_input, encoder_self_attention_bias, hparams), ed
Example #24
| Source File: transformer_revnet.py From tensor2tensor with Apache License 2.0 | 5 votes |
|
def body(self, features):
hparams = self._hparams
targets = features["targets"]
inputs = features["inputs"]
target_space = features["target_space_id"]
inputs = common_layers.flatten4d3d(inputs)
targets = common_layers.flatten4d3d(targets)
(encoder_input, encoder_self_attention_bias,
encoder_decoder_attention_bias) = (transformer.transformer_prepare_encoder(
inputs, target_space, hparams))
(decoder_input,
decoder_self_attention_bias) = transformer.transformer_prepare_decoder(
targets, hparams)
encoder_input = tf.nn.dropout(encoder_input,
1.0 - hparams.layer_prepostprocess_dropout)
decoder_input = tf.nn.dropout(decoder_input,
1.0 - hparams.layer_prepostprocess_dropout)
encoder_output = transformer_revnet_encoder(
encoder_input, encoder_self_attention_bias, hparams)
decoder_output = transformer_revnet_decoder(
decoder_input, encoder_output, decoder_self_attention_bias,
encoder_decoder_attention_bias, hparams)
decoder_output = tf.expand_dims(decoder_output, 2)
return decoder_output
Example #25
| Source File: transformer_nat.py From fine-lm with MIT License | 5 votes |
|
def encode(x, x_space, hparams, name):
"""Transformer preparations and encoder."""
with tf.variable_scope(name):
(encoder_input, encoder_self_attention_bias,
ed) = transformer.transformer_prepare_encoder(x, x_space, hparams)
encoder_input = tf.nn.dropout(encoder_input, 1.0 - hparams.dropout)
return transformer.transformer_encoder(
encoder_input, encoder_self_attention_bias, hparams), ed
Example #26
| Source File: transformer_vae.py From fine-lm with MIT License | 5 votes |
|
def encode(x, x_space, hparams, name):
"""Transformer preparations and encoder."""
with tf.variable_scope(name):
(encoder_input, encoder_self_attention_bias,
ed) = transformer.transformer_prepare_encoder(x, x_space, hparams)
encoder_input = tf.nn.dropout(encoder_input, 1.0 - hparams.dropout)
return transformer.transformer_encoder(
encoder_input, encoder_self_attention_bias, hparams), ed
Example #27
| Source File: nas_model.py From BERT with Apache License 2.0 | 4 votes |
|
def encode(self, inputs, target_space, hparams, features=None, losses=None):
"""Encode inputs using _encoder().
This performs the same way as transformer.Transformer.encode with the
encoder portion replaced with _encoder().
Args:
inputs: Input [batch_size, input_length, input_height, hidden_dim] tensor
which will be flattened along the two spatial dimensions.
target_space: scalar, target space ID.
hparams: Hyperparmeters for model.
features: Optionally pass the entire features dictionary as well. This is
needed now for "packed" datasets.
losses: Unused list of losses.
Returns:
Tuple of:
encoder_output: Encoder representation.
[batch_size, input_length, hidden_dim]
encoder_decoder_attention_bias: Bias and mask weights for
encodre-decoder attention. [batch_size, input_length]
Raises:
ValueError: If encoder type not found.
"""
inputs = common_layers.flatten4d3d(inputs)
encoder_input, self_attention_bias, encoder_decoder_attention_bias = (
transformer.transformer_prepare_encoder(
inputs, target_space, hparams, features=features))
encoder_input = tf.nn.dropout(encoder_input,
1.0 - hparams.layer_prepostprocess_dropout)
encoder_output = self._encoder(
encoder_input,
self_attention_bias,
hparams,
nonpadding=transformer.features_to_nonpadding(features, "inputs"),
save_weights_to=self.attention_weights)
return encoder_output, encoder_decoder_attention_bias
Example #28
| Source File: nas_model.py From tensor2tensor with Apache License 2.0 | 4 votes |
|
def encode(self, inputs, target_space, hparams, features=None, losses=None):
"""Encode inputs using _encoder().
This performs the same way as transformer.Transformer.encode with the
encoder portion replaced with _encoder().
Args:
inputs: Input [batch_size, input_length, input_height, hidden_dim] tensor
which will be flattened along the two spatial dimensions.
target_space: scalar, target space ID.
hparams: Hyperparmeters for model.
features: Optionally pass the entire features dictionary as well. This is
needed now for "packed" datasets.
losses: Unused list of losses.
Returns:
Tuple of:
encoder_output: Encoder representation.
[batch_size, input_length, hidden_dim]
encoder_decoder_attention_bias: Bias and mask weights for
encodre-decoder attention. [batch_size, input_length]
Raises:
ValueError: If encoder type not found.
"""
inputs = common_layers.flatten4d3d(inputs)
encoder_input, self_attention_bias, encoder_decoder_attention_bias = (
transformer.transformer_prepare_encoder(
inputs, target_space, hparams, features=features))
encoder_input = tf.nn.dropout(encoder_input,
1.0 - hparams.layer_prepostprocess_dropout)
encoder_output = self._encoder(
encoder_input,
self_attention_bias,
hparams,
nonpadding=transformer.features_to_nonpadding(features, "inputs"),
save_weights_to=self.attention_weights)
return encoder_output, encoder_decoder_attention_bias
Example #29
| Source File: universal_transformer.py From fine-lm with MIT License | 4 votes |
|
def encode(self, inputs, target_space, hparams, features=None, losses=None):
"""Encode Universal Transformer inputs.
It is similar to "transformer.encode", but it uses
"universal_transformer_util.universal_transformer_encoder" instead of
"transformer.transformer_encoder".
Args:
inputs: Transformer inputs [batch_size, input_length, input_height,
hidden_dim] which will be flattened along the two spatial dimensions.
target_space: scalar, target space ID.
hparams: hyperparmeters for model.
features: optionally pass the entire features dictionary as well.
This is needed now for "packed" datasets.
losses: Unused.
Returns:
Tuple of:
encoder_output: Encoder representation.
[batch_size, input_length, hidden_dim]
encoder_decoder_attention_bias: Bias and mask weights for
encoder-decoder attention. [batch_size, input_length]
encoder_extra_output: which is extra encoder output used in some
variants of the model (e.g. in ACT, to pass the ponder-time to body)
"""
del losses
inputs = common_layers.flatten4d3d(inputs)
encoder_input, self_attention_bias, encoder_decoder_attention_bias = (
transformer.transformer_prepare_encoder(
inputs, target_space, hparams, features=features))
encoder_input = tf.nn.dropout(encoder_input,
1.0 - hparams.layer_prepostprocess_dropout)
(encoder_output, encoder_extra_output) = (
universal_transformer_util.universal_transformer_encoder(
encoder_input,
self_attention_bias,
hparams,
nonpadding=transformer.features_to_nonpadding(features, "inputs"),
save_weights_to=self.attention_weights))
return encoder_output, encoder_decoder_attention_bias, encoder_extra_output
Example #30
| Source File: universal_transformer_modified.py From Graph-Transformer with Apache License 2.0 | 4 votes |
|
def encode(self, inputs, target_space, hparams, features=None, losses=None):
"""Encode Universal Transformer inputs.
It is similar to "transformer.encode", but it uses
"universal_transformer_util.universal_transformer_encoder" instead of
"transformer.transformer_encoder".
Args:
inputs: Transformer inputs [batch_size, input_length, input_height,
hidden_dim] which will be flattened along the two spatial dimensions.
target_space: scalar, target space ID.
hparams: hyperparmeters for model.
features: optionally pass the entire features dictionary as well.
This is needed now for "packed" datasets.
losses: Unused.
Returns:
Tuple of:
encoder_output: Encoder representation.
[batch_size, input_length, hidden_dim]
encoder_decoder_attention_bias: Bias and mask weights for
encoder-decoder attention. [batch_size, input_length]
encoder_extra_output: which is extra encoder output used in some
variants of the model (e.g. in ACT, to pass the ponder-time to body)
"""
del losses
inputs = common_layers.flatten4d3d(inputs)
encoder_input, self_attention_bias, encoder_decoder_attention_bias = (
transformer.transformer_prepare_encoder(
inputs, target_space, hparams, features=features))
encoder_input = tf.nn.dropout(encoder_input,
1.0 - hparams.layer_prepostprocess_dropout)
(encoder_output, encoder_extra_output) = (
universal_transformer_util.universal_transformer_encoder(
encoder_input,
self_attention_bias,
hparams,
nonpadding=transformer.features_to_nonpadding(features, "inputs"),
save_weights_to=self.attention_weights))
return encoder_output, encoder_decoder_attention_bias, encoder_extra_output
