Python tensorflow.python.layers.core.Dense() Examples

def build(self, input_shape):
        input_shape = tensor_shape.TensorShape(input_shape)
        if input_shape[-1].value is None:
            raise ValueError('The last dimension of the inputs to `Dense` '
                             'should be defined. Found `None`.')
        self.input_spec = base.InputSpec(min_ndim=2,
                                         axes={-1: input_shape[-1].value})

        self.kernel = self._build_kernel(input_shape)

        if self.use_bias:
            self.bias = self.add_variable('bias',
                                          shape=[self.units],
                                          initializer=self.bias_initializer,
                                          regularizer=self.bias_regularizer,
                                          constraint=self.bias_constraint,
                                          dtype=self.dtype,
                                          trainable=True)
        else:
            self.bias = None
        self.built = True 

def _maybe_add_dense_layers(self):
        r"""
        Optionally passes self._input through several Fully Connected (Dense) layers
        with the configuration defined by the self._input_dense_layers tuple

        Returns
        -------
        The output of the network of Dense layers
        """
        layer_inputs = self._inputs
        if self._hparams.input_dense_layers[0] > 0:

            fc = [Dense(units,
                        activation=tf.nn.selu,
                        use_bias=False,
                        kernel_initializer=tf.variance_scaling_initializer(),
                        kernel_regularizer=tf.contrib.layers.l2_regularizer(scale=0.0001))
                  for units in self._hparams.input_dense_layers]

            for layer in fc:
                layer_inputs = layer(layer_inputs)
        else:
            pass
        return layer_inputs 

def _init_decoder(self):
        with tf.variable_scope("Decoder"):
            self._decoder_cells = build_rnn_layers(
                cell_type=self._hparams.cell_type,
                num_units_per_layer=self._hparams.decoder_units_per_layer,
                use_dropout=self._hparams.use_dropout,
                dropout_probability=self._hparams.decoder_dropout_probability,
                mode=self._mode,
                dtype=self._hparams.dtype,
            )

            self._decoder_initial_state = self._decoder_cells.zero_state(
                batch_size=self._batch_size, dtype=self._hparams.dtype)

            self._dense_layer = Dense(
                self._vocab_size,
                name='my_dense',
                dtype=self._hparams.dtype)

            self._build_decoder_train()  # used for both training and evaluation 

def monkeypatch_tf_layers():
    if get_tf_version_tuple() < (1, 4):
        if not hasattr(tf.layers, 'Dense'):
            from tensorflow.python.layers.core import Dense
            tf.layers.Dense = Dense

            from tensorflow.python.layers.normalization import BatchNormalization
            tf.layers.BatchNormalization = BatchNormalization

            from tensorflow.python.layers.convolutional import Conv2DTranspose, Conv2D
            tf.layers.Conv2DTranspose = Conv2DTranspose
            tf.layers.Conv2D = Conv2D

            from tensorflow.python.layers.pooling import MaxPooling2D, AveragePooling2D
            tf.layers.MaxPooling2D = MaxPooling2D
            tf.layers.AveragePooling2D = AveragePooling2D 

def _project_lstm_state_tuple(state_tuple, num_units):
    r"""
    Concatenates all the `c` and `h` members from a list of `LSTMStateTuple`
      and projects them to a space of dimension `num_units`
    Args:
        state_tuple: a list of `LSTMStateTuple` objects
        num_units: output dimension

    Returns:
        projected_state: a single `LSTMStateTuple` with `c` and `h` of dimension `num_units`
    """
    state_proj_layer = Dense(num_units, name='state_projection', use_bias=False)

    cat_c = tf.concat([state.c for state in state_tuple], axis=-1)
    cat_h = tf.concat([state.h for state in state_tuple], axis=-1)

    proj_c = state_proj_layer(cat_c)
    proj_h = state_proj_layer(cat_h)

    projected_state = tf.contrib.rnn.LSTMStateTuple(c=proj_c, h=proj_h)

    return projected_state 

def monkeypatch_tf_layers():
    if get_tf_version_tuple() < (1, 4):
        if not hasattr(tf.layers, 'Dense'):
            from tensorflow.python.layers.core import Dense
            tf.layers.Dense = Dense

            from tensorflow.python.layers.normalization import BatchNormalization
            tf.layers.BatchNormalization = BatchNormalization

            from tensorflow.python.layers.convolutional import Conv2DTranspose, Conv2D
            tf.layers.Conv2DTranspose = Conv2DTranspose
            tf.layers.Conv2D = Conv2D

            from tensorflow.python.layers.pooling import MaxPooling2D, AveragePooling2D
            tf.layers.MaxPooling2D = MaxPooling2D
            tf.layers.AveragePooling2D = AveragePooling2D 

def monkeypatch_tf_layers():
    if get_tf_version_tuple() < (1, 4):
        if not hasattr(tf.layers, 'Dense'):
            from tensorflow.python.layers.core import Dense
            tf.layers.Dense = Dense

            from tensorflow.python.layers.normalization import BatchNormalization
            tf.layers.BatchNormalization = BatchNormalization

            from tensorflow.python.layers.convolutional import Conv2DTranspose, Conv2D
            tf.layers.Conv2DTranspose = Conv2DTranspose
            tf.layers.Conv2D = Conv2D

            from tensorflow.python.layers.pooling import MaxPooling2D, AveragePooling2D
            tf.layers.MaxPooling2D = MaxPooling2D
            tf.layers.AveragePooling2D = AveragePooling2D 

def get_config(self):
    config = {
        'units': self.units,
        'activation': activations.serialize(self.activation),
        'use_bias': self.use_bias,
        'kernel_initializer': initializers.serialize(self.kernel_initializer),
        'bias_initializer': initializers.serialize(self.bias_initializer),
        'kernel_regularizer': regularizers.serialize(self.kernel_regularizer),
        'bias_regularizer': regularizers.serialize(self.bias_regularizer),
        'activity_regularizer':
            regularizers.serialize(self.activity_regularizer),
        'kernel_constraint': constraints.serialize(self.kernel_constraint),
        'bias_constraint': constraints.serialize(self.bias_constraint)
    }
    base_config = super(Dense, self).get_config()
    return dict(list(base_config.items()) + list(config.items())) 

def __init__(self,
                 num_mixtures,
                 memory,
                 memory_sequence_length=None,
                 check_inner_dims_defined=True,
                 score_mask_value=None,
                 name='GmmAttention'):

        self.dtype = memory.dtype
        self.num_mixtures = num_mixtures
        self.query_layer = tf.layers.Dense(3 * num_mixtures, name='gmm_query_layer', use_bias=True, dtype=self.dtype)

        with tf.name_scope(name, 'GmmAttentionMechanismInit'):
            if score_mask_value is None:
                score_mask_value = 0.
            self._maybe_mask_score = functools.partial(
                _maybe_mask_score,
                memory_sequence_length=memory_sequence_length,
                score_mask_value=score_mask_value)
            self._value = _prepare_memory(
                memory, memory_sequence_length, check_inner_dims_defined)
            self._batch_size = (
                self._value.shape[0].value or tf.shape(self._value)[0])
            self._alignments_size = (
                    self._value.shape[1].value or tf.shape(self._value)[1]) 

def build_rnn(self):
        self.initial_state = tf.cond(
            self.beam_search_decoding, lambda: seq2seq.tile_batch(
                self.features["state"], self.hparams.beam_width),
            lambda: self.features["state"], name="initial_state")
        self.build_embeddings()
        cell_list = self.build_deep_cell(return_raw_list=True)
        if self.hparams.use_attention:
            cell_list[-1] = self.build_attention(cell_list[-1])
            if self.hparams.depth > 1:
                self.initial_state[-1] = final_cell.zero_state(batch_size=self.batch_size)
            else:
                self.initial_state = final_cell.zero_state(batch_size=self.batch_size)
        with tf.name_scope('rnn_cell'):
            self.cell = self.build_deep_cell(cell_list)
        self.output_layer = Dense(self.vocab.size(), name='output_layer') 

def get_config(self):
    config = {
        'units': self.units,
        'activation': activations.serialize(self.activation),
        'use_bias': self.use_bias,
        'kernel_initializer': initializers.serialize(self.kernel_initializer),
        'bias_initializer': initializers.serialize(self.bias_initializer),
        'kernel_regularizer': regularizers.serialize(self.kernel_regularizer),
        'bias_regularizer': regularizers.serialize(self.bias_regularizer),
        'activity_regularizer':
            regularizers.serialize(self.activity_regularizer),
        'kernel_constraint': constraints.serialize(self.kernel_constraint),
        'bias_constraint': constraints.serialize(self.bias_constraint)
    }
    base_config = super(Dense, self).get_config()
    return dict(list(base_config.items()) + list(config.items())) 

def monkeypatch_tf_layers():
    if get_tf_version_tuple() < (1, 4):
        if not hasattr(tf.layers, 'Dense'):
            from tensorflow.python.layers.core import Dense
            tf.layers.Dense = Dense

            from tensorflow.python.layers.normalization import BatchNormalization
            tf.layers.BatchNormalization = BatchNormalization

            from tensorflow.python.layers.convolutional import Conv2DTranspose, Conv2D
            tf.layers.Conv2DTranspose = Conv2DTranspose
            tf.layers.Conv2D = Conv2D

            from tensorflow.python.layers.pooling import MaxPooling2D, AveragePooling2D
            tf.layers.MaxPooling2D = MaxPooling2D
            tf.layers.AveragePooling2D = AveragePooling2D 

def __att_decode(self, helper, rnn_features, scope, reuse=None):
        """
        Attention decode part
        :param helper: train or inference
        :param rnn_features: encoded features
        :param scope: name scope
        :param reuse: reuse or not
        :return: attention decode output
        """
        with tf.variable_scope(scope, reuse=reuse):
            if self.attention_mode == 1:
                attention_mechanism = tf.contrib.seq2seq.LuongAttention(num_units=self.rnn_units,
                                                                        memory=rnn_features)
            else:
                attention_mechanism = tf.contrib.seq2seq.BahdanauAttention(num_units=self.rnn_units,
                                                                           memory=rnn_features)

            cell = tf.contrib.rnn.GRUCell(num_units=self.rnn_units)
            attn_cell = tf.contrib.seq2seq.AttentionWrapper(cell, attention_mechanism,
                                                            attention_layer_size=self.rnn_units,
                                                            output_attention=True)
            output_layer = Dense(units=self.vocab_att_size)

            decoder = tf.contrib.seq2seq.BasicDecoder(
                cell=attn_cell, helper=helper,
                initial_state=attn_cell.zero_state(dtype=tf.float32, batch_size=self.batch_size),
                output_layer=output_layer)

            att_outputs = tf.contrib.seq2seq.dynamic_decode(
                decoder=decoder, output_time_major=False,
                impute_finished=True, maximum_iterations=self.max_dec_iteration)

            return att_outputs 

def decode(helper, memory, scope, reuse=None):
    with tf.variable_scope(scope, reuse=reuse):
        attention_mechanism = tf.contrib.seq2seq.LuongAttention(num_units=RNN_UNITS, memory=memory)
        cell = tf.contrib.rnn.GRUCell(num_units=RNN_UNITS)
        attn_cell = tf.contrib.seq2seq.AttentionWrapper(cell, attention_mechanism, attention_layer_size=RNN_UNITS, output_attention=True)
        output_layer = Dense(units=VOCAB_SIZE)

        decoder = tf.contrib.seq2seq.BasicDecoder(
            cell=attn_cell, helper=helper,
            initial_state=attn_cell.zero_state(dtype=tf.float32, batch_size=BATCH_SIZE),
            output_layer=output_layer)
        outputs = tf.contrib.seq2seq.dynamic_decode(
            decoder=decoder, output_time_major=False,
            impute_finished=True, maximum_iterations=MAXIMUM__DECODE_ITERATIONS)
        return outputs 

def build(self, input_shape):
    super(Dense, self).build(input_shape)
    # TODO(fchollet): move weight constraint support to core layers.
    if self.kernel_constraint:
      self.constraints[self.kernel] = self.kernel_constraint
    if self.use_bias and self.bias_constraint:
      self.constraints[self.bias] = self.bias_constraint 

def build_latent_space(self):
        with tf.name_scope("latent_space"):
            self.z_mean = Dense(self.latent_dim, name='z_mean')(self.h_N)
            self.z_log_sigma = Dense(self.latent_dim, name='z_log_sigma')(self.h_N)

            self.z_vector = tf.identity(self.sample_gaussian(), name='z_vector') 

def _init_decoder(self):
        r"""
        Builds the decoder blocks: the cells, the initial state, the output projection layer,
        the decoding algorithm, the attention layers and the trainining optimiser
        """

        with tf.variable_scope("Decoder"):

            self._decoder_cells = build_rnn_layers(
                cell_type=self._hparams.cell_type,
                num_units_per_layer=self._hparams.decoder_units_per_layer,
                use_dropout=self._hparams.use_dropout,
                dropout_probability=self._hparams.decoder_dropout_probability,
                mode=self._mode,
                dtype=self._hparams.dtype,
            )

            self._construct_decoder_initial_state()

            self._dense_layer = Dense(self._vocab_size,
                                      name='my_dense',
                                      dtype=self._hparams.dtype)

            if self._mode == 'train':
                self._build_decoder_train()
            else:
                if self._hparams.decoding_algorithm == 'greedy':
                    self._build_decoder_test_greedy()
                elif self._hparams.decoding_algorithm == 'beam_search':
                    self._build_decoder_test_beam_search()
                else:
                    raise Exception('The only supported algorithms are `greedy` and `beam_search`') 

def _init_decoder(self):
        r"""
                Instantiates the seq2seq decoder
                :return:
                """

        with tf.variable_scope("Decoder"):

            self._decoder_cells = build_rnn_layers(
                cell_type=self._hparams.cell_type,
                num_units_per_layer=self._hparams.decoder_units_per_layer,
                use_dropout=self._hparams.use_dropout,
                dropout_probability=self._hparams.decoder_dropout_probability,
                mode=self._mode,
                dtype=self._hparams.dtype,
            )

            self._construct_decoder_initial_state()

            self._dense_layer = Dense(self._vocab_size,
                                      name='my_dense',
                                      dtype=self._hparams.dtype)

            if self._mode == 'train':
                self._build_decoder_train()
            else:
                if self._hparams.decoding_algorithm == 'greedy':
                    self._build_decoder_greedy()
                elif self._hparams.decoding_algorithm == 'beam_search':
                    self._build_decoder_beam_search()
                else:
                    raise Exception('The only supported algorithms are `greedy` and `beam_search`') 

def __create_decoder_cell(self):

        self.decoder_cell = tf.nn.rnn_cell.GRUCell(self.config.DECODER_RNN_HIDDEN_SIZE)

        # fully connected layer to change size of Encoder Last state to Decoder Hidden size
        decoder_hidden_state_reshape = Dense(self.config.DECODER_RNN_HIDDEN_SIZE)

        self.decoder_initial_state = (decoder_hidden_state_reshape(self.encoder_last_state), ) 

def _project_lstm_state_tuple(state_tuple, num_units):

    state_proj_layer = Dense(num_units, name='state_projection', use_bias=False)

    cat_c = tf.concat([state.c for state in state_tuple], axis=-1)
    cat_h = tf.concat([state.h for state in state_tuple], axis=-1)

    proj_c = state_proj_layer(cat_c)
    proj_h = state_proj_layer(cat_h)

    projected_state = tf.contrib.rnn.LSTMStateTuple(c=proj_c, h=proj_h)

    return projected_state 

def self_attention_2(self, inputs, name):
        """

        :param inputs_a: audio input (B, T, dim)
        :param inputs_v: video input (B, T, dim)
        :param inputs_t: text input (B, T, dim)
        :param name: scope name
        :return:
        """

        t = inputs.get_shape()[1].value
        share_param = True
        hidden_size = inputs.shape[-1].value  # D value - hidden size of the RNN layer
        if share_param:
            scope_name = 'self_attn_2'
        else:
            scope_name = 'self_attn_2' + name
        # print(scope_name)
        # inputs = tf.transpose(inputs, [2, 0, 1, 3])
        # dense = Dense(hidden_size)
        # init1 = tf.random_normal_initializer(seed=self.seed, dtype=tf.float32,stddev=0.01)
        attention_size = hidden_size
        w_omega = tf.Variable(tf.random_normal([hidden_size, attention_size], stddev=0.01, seed=self.seed))
        b_omega = tf.Variable(tf.random_normal([attention_size], stddev=0.01, seed=self.seed))
        # dense_attention_2 = Dense(attention_size, activation=None,kernel_initializer=init1,kernel_regularizer=tf.contrib.layers.l2_regularizer(0.001))
        params = {'w_omega': w_omega,
                  'b_omega': b_omega,
                  # 'dense': dense_attention_2
                  }
        with tf.variable_scope(scope_name, reuse=tf.AUTO_REUSE):
            outputs = []
            for x in range(t):
                t_x = inputs[:, x, :]

                output = self.attention(inputs, t_x, hidden_size, params, self.mask)  # (b, d)
                outputs.append(output)

            final_output = tf.concat(outputs, axis=1)
            return final_output 

def __init__(self, cell, helper, initial_state, output_layer=None):
        """Initialize BasicVectorDecoder.

        Args:
            cell: An `RNNCell` instance.
            helper: A `Helper` instance.
            initial_state: A (possibly nested tuple of...) tensors and TensorArrays.
                The initial state of the RNNCell.
            output_layer:An instance of `tf.layers.Layer`, i.e., `tf.layers.Dense`.
                If not provided, use 1 fc layer.

        Raises:
            TypeError: if `cell`, `helper` or `output_layer` have an incorrect type.
        """
        if not rnn_cell_impl._like_rnncell(cell):  # pylint: disable=protected-access
            raise TypeError("cell must be an RNNCell, received: %s" % type(cell))
        if not isinstance(helper, helper_py.Helper):
            raise TypeError("helper must be a Helper, received: %s" % type(helper))
        if (output_layer is not None and not isinstance(output_layer, layers_base.Layer)):
            raise TypeError(
                "output_layer must be a Layer, received: %s" % type(output_layer))
        self._cell = cell
        self._helper = helper
        self._initial_state = initial_state
        if output_layer is None:
            self._output_layer = layer_core.Dense(2, use_bias=True,
                                                  name="stop_predictor")
        else:
            self._output_layer = output_layer 

def __init__(
      self,
      attention_units,
      query_dim,
      name="location",
      dtype=None,
      **kwargs
  ):
    super(ZhaopengLocationLayer, self).__init__(name=name, **kwargs)
    self.vbeta = variable_scope.get_variable(
        "location_attention_vbeta", [query_dim], dtype=dtypes.float32)
    self.location_dense = layers_core.Dense(
        name="{}_dense".format(name), units=attention_units, use_bias=False
    ) 

def build(self, hparams, output_depth, is_training=True):
    if hparams.use_cudnn and hparams.residual_decoder:
      raise ValueError('Residual connections not supported in cuDNN.')

    self._is_training = is_training

    tf.logging.info('\nDecoder Cells:\n'
                    '  units: %s\n',
                    hparams.dec_rnn_size)

    self._sampling_probability = lstm_utils.get_sampling_probability(
        hparams, is_training)
    self._output_depth = output_depth
    self._output_layer = layers_core.Dense(
        output_depth, name='output_projection')
    self._dec_cell = lstm_utils.rnn_cell(
        hparams.dec_rnn_size, hparams.dropout_keep_prob,
        hparams.residual_decoder, is_training)
    if hparams.use_cudnn:
      self._cudnn_dec_lstm = lstm_utils.cudnn_lstm_layer(
          hparams.dec_rnn_size, hparams.dropout_keep_prob, is_training,
          name_or_scope='decoder')
    else:
      self._cudnn_dec_lstm = None

    self.max_length =tf.constant(hparams.max_seq_len, tf.int32) 

def build(self, hparams, output_depth, is_training=False):
    self._nade = Nade(
        output_depth, hparams.nade_num_hidden, name='decoder/nade')
    super(MultiLabelRnnNadeDecoder, self).build(
        hparams, output_depth, is_training)
    # Overwrite output layer for NADE parameterization.
    self._output_layer = layers_core.Dense(
        self._nade.num_hidden + output_depth, name='output_projection') 

def build_contextual_encoding(self, inputs_, name=""):
        embedded = tf.nn.embedding_lookup(
            params=self.token_embeddings, ids=inputs_)
        if self.hparams.use_embedding_projection:
            projection = Dense(self.hparams.hidden_units,
                               name=name+'_input_projection')
            embedded = projection(embedded)
        length = tf.cast(tf.count_nonzero(
            self.query - self.vocab.end_token_id, -1), tf.int32)
        return self.build_cell(embedded, length, name + "/contextual") 

def build_obj(self):
        output_layer = Dense(self.tgt_vocab.size(), name='output_projection')
        self.logits = output_layer(self.outputs) 

def build_contextual_encoding(self, inputs_, name=""):
        embedded = tf.nn.embedding_lookup(
            params=self.token_embeddings, ids=inputs_)
        if self.hparams.use_embedding_projection:
            projection = Dense(self.hparams.hidden_units,
                               name=name+'_input_projection')
            embedded = projection(embedded)
        length = tf.cast(tf.count_nonzero(
            self.query - self.vocab.end_token_id, -1), tf.int32)
        return self.build_cell(embedded, length, name + "/contextual") 

def _attention_input_feeding(self, input_feed):
        if self.hparams.attention_input_feeding:
            self.attention_input_layer = Dense(self.hparams.hidden_units, name='attention_input_layer')
            return self.attention_input_layer(tf.concat([input_feed, attention], -1))
        else:
            return input_feed 

def __init__(self,
               units,
               activation=None,
               use_bias=True,
               kernel_initializer='glorot_uniform',
               bias_initializer='zeros',
               kernel_regularizer=None,
               bias_regularizer=None,
               activity_regularizer=None,
               kernel_constraint=None,
               bias_constraint=None,
               **kwargs):
    if 'input_shape' not in kwargs and 'input_dim' in kwargs:
      kwargs['input_shape'] = (kwargs.pop('input_dim'),)

    # Inheritance call order:
    # 1) tf.layers.Dense, 2) keras.layers.Layer, 3) tf.layers.Layer
    super(Dense, self).__init__(
        units,
        activation=activations.get(activation),
        use_bias=use_bias,
        kernel_initializer=initializers.get(kernel_initializer),
        bias_initializer=initializers.get(bias_initializer),
        kernel_regularizer=regularizers.get(kernel_regularizer),
        bias_regularizer=regularizers.get(bias_regularizer),
        activity_regularizer=regularizers.get(activity_regularizer),
        kernel_constraint=constraints.get(kernel_constraint),
        bias_constraint=constraints.get(bias_constraint),
        **kwargs)
    self.supports_masking = True