Python inputs.inputs() Examples

def language_model_graph(self, compute_loss=True):
    """Constructs LM graph from inputs to LM loss.

    * Caches the VatxtInput object in `self.lm_inputs`
    * Caches tensors: `lm_embedded`

    Args:
      compute_loss: bool, whether to compute and return the loss or stop after
        the LSTM computation.

    Returns:
      loss: scalar float.
    """
    inputs = _inputs('train', pretrain=True)
    self.lm_inputs = inputs
    return self._lm_loss(inputs, compute_loss=compute_loss) 

def _lm_loss(self,
               inputs,
               emb_key='lm_embedded',
               lstm_layer='lstm',
               lm_loss_layer='lm_loss',
               loss_name='lm_loss',
               compute_loss=True):
    embedded = self.layers['embedding'](inputs.tokens)
    self.tensors[emb_key] = embedded
    lstm_out, next_state = self.layers[lstm_layer](embedded, inputs.state,
                                                   inputs.length)
    if compute_loss:
      loss = self.layers[lm_loss_layer](
          [lstm_out, inputs.labels, inputs.weights])
      with tf.control_dependencies([inputs.save_state(next_state)]):
        loss = tf.identity(loss)
        tf.summary.scalar(loss_name, loss)

      return loss 

def language_model_graph(self, compute_loss=True):
    """Constructs LM graph from inputs to LM loss.

    * Caches the VatxtInput object in `self.lm_inputs`
    * Caches tensors: `lm_embedded`

    Args:
      compute_loss: bool, whether to compute and return the loss or stop after
        the LSTM computation.

    Returns:
      loss: scalar float.
    """
    inputs = _inputs('train', pretrain=True)
    self.lm_inputs = inputs
    return self._lm_loss(inputs, compute_loss=compute_loss) 

def _lm_loss(self,
               inputs,
               emb_key='lm_embedded',
               lstm_layer='lstm',
               lm_loss_layer='lm_loss',
               loss_name='lm_loss',
               compute_loss=True):
    embedded = self.layers['embedding'](inputs.tokens)
    self.tensors[emb_key] = embedded
    lstm_out, next_state = self.layers[lstm_layer](embedded, inputs.state,
                                                   inputs.length)
    if compute_loss:
      loss = self.layers[lm_loss_layer](
          [lstm_out, inputs.labels, inputs.weights])
      with tf.control_dependencies([inputs.save_state(next_state)]):
        loss = tf.identity(loss)
        tf.summary.scalar(loss_name, loss)

      return loss 

def language_model_graph(self, compute_loss=True):
    """Constructs LM graph from inputs to LM loss.

    * Caches the VatxtInput object in `self.lm_inputs`
    * Caches tensors: `lm_embedded`

    Args:
      compute_loss: bool, whether to compute and return the loss or stop after
        the LSTM computation.

    Returns:
      loss: scalar float.
    """
    inputs = _inputs('train', pretrain=True)
    self.lm_inputs = inputs
    return self._lm_loss(inputs, compute_loss=compute_loss) 

def _lm_loss(self,
               inputs,
               emb_key='lm_embedded',
               lstm_layer='lstm',
               lm_loss_layer='lm_loss',
               loss_name='lm_loss',
               compute_loss=True):
    embedded = self.layers['embedding'](inputs.tokens)
    self.tensors[emb_key] = embedded
    lstm_out, next_state = self.layers[lstm_layer](embedded, inputs.state,
                                                   inputs.length)
    if compute_loss:
      loss = self.layers[lm_loss_layer](
          [lstm_out, inputs.labels, inputs.weights])
      with tf.control_dependencies([inputs.save_state(next_state)]):
        loss = tf.identity(loss)
        tf.summary.scalar(loss_name, loss)

      return loss 

def language_model_graph(self, compute_loss=True):
    """Constructs LM graph from inputs to LM loss.

    * Caches the VatxtInput object in `self.lm_inputs`
    * Caches tensors: `lm_embedded`

    Args:
      compute_loss: bool, whether to compute and return the loss or stop after
        the LSTM computation.

    Returns:
      loss: scalar float.
    """
    inputs = _inputs('train', pretrain=True)
    self.lm_inputs = inputs
    return self._lm_loss(inputs, compute_loss=compute_loss) 

def language_model_graph(self, compute_loss=True):
    """Constructs LM graph from inputs to LM loss.

    * Caches the VatxtInput object in `self.lm_inputs`
    * Caches tensors: `lm_embedded`

    Args:
      compute_loss: bool, whether to compute and return the loss or stop after
        the LSTM computation.

    Returns:
      loss: scalar float.
    """
    inputs = _inputs('train', pretrain=True)
    self.lm_inputs = inputs
    return self._lm_loss(inputs, compute_loss=compute_loss) 

def language_model_graph(self, compute_loss=True):
    """Constructs LM graph from inputs to LM loss.

    * Caches the VatxtInput object in `self.lm_inputs`
    * Caches tensors: `lm_embedded`

    Args:
      compute_loss: bool, whether to compute and return the loss or stop after
        the LSTM computation.

    Returns:
      loss: scalar float.
    """
    inputs = _inputs('train', pretrain=True)
    self.lm_inputs = inputs
    return self._lm_loss(inputs, compute_loss=compute_loss) 

def _lm_loss(self,
               inputs,
               emb_key='lm_embedded',
               lstm_layer='lstm',
               lm_loss_layer='lm_loss',
               loss_name='lm_loss',
               compute_loss=True):
    embedded = self.layers['embedding'](inputs.tokens)
    self.tensors[emb_key] = embedded
    lstm_out, next_state = self.layers[lstm_layer](embedded, inputs.state,
                                                   inputs.length)
    if compute_loss:
      loss = self.layers[lm_loss_layer](
          [lstm_out, inputs.labels, inputs.weights])
      with tf.control_dependencies([inputs.save_state(next_state)]):
        loss = tf.identity(loss)
        tf.summary.scalar(loss_name, loss)

      return loss 

def _lm_loss(self,
               inputs,
               emb_key='lm_embedded',
               lstm_layer='lstm',
               lm_loss_layer='lm_loss',
               loss_name='lm_loss',
               compute_loss=True):
    embedded = self.layers['embedding'](inputs.tokens)
    self.tensors[emb_key] = embedded
    lstm_out, next_state = self.layers[lstm_layer](embedded, inputs.state,
                                                   inputs.length)
    if compute_loss:
      loss = self.layers[lm_loss_layer](
          [lstm_out, inputs.labels, inputs.weights])
      with tf.control_dependencies([inputs.save_state(next_state)]):
        loss = tf.identity(loss)
        tf.summary.scalar(loss_name, loss)

      return loss 

def _activation_summary(self, x):
        """Helper to create summaries for activations.
        Creates a summary that provides a histogram of activations.
        Creates a summary that measure the sparsity of activations.
        Args:
            x: Tensor
        Returns:
            nothing
        """
        # Remove 'tower_[0-9]/' from the name in case this is a multi-GPU training
        # session. This helps the clarity of presentation on tensorboard.
        # Error: these summaries cause high classifier error!!!
        # All inputs to node MergeSummary/MergeSummary must be from the same frame.

        # tensor_name = re.sub('%s_[0-9]*/' % "tower", '', x.op.name)
        # tf.histogram_summary(tensor_name + '/activations', x)
        # tf.scalar_summary(tensor_name + '/sparsity', tf.nn.zero_fraction(x)) 

def _lm_loss(self,
               inputs,
               emb_key='lm_embedded',
               lstm_layer='lstm',
               lm_loss_layer='lm_loss',
               loss_name='lm_loss',
               compute_loss=True):
    embedded = self.layers['embedding'](inputs.tokens)
    self.tensors[emb_key] = embedded
    lstm_out, next_state = self.layers[lstm_layer](embedded, inputs.state,
                                                   inputs.length)
    if compute_loss:
      loss = self.layers[lm_loss_layer](
          [lstm_out, inputs.labels, inputs.weights])
      with tf.control_dependencies([inputs.save_state(next_state)]):
        loss = tf.identity(loss)
        tf.summary.scalar(loss_name, loss)

      return loss 

def loss(logits, labels):
    """Add L2Loss to all the trainable variables.
    Add summary for "Loss" and "Loss/avg".
    Args:
        logits: Logits from inference().
        labels: Labels from distorted_inputs or inputs(). 1-D tensor
                of shape [batch_size]
    Returns:
        Loss tensor of type float.
    """
    # Calculate the average cross entropy loss across the batch.
    labels = tf.cast(labels, tf.int64)
    cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(
        logits,
        labels,
        name='cross_entropy_per_example')
    cross_entropy_mean = tf.reduce_mean(cross_entropy, name='cross_entropy')
    tf.add_to_collection('losses', cross_entropy_mean)

    # The total loss is defined as the cross entropy loss plus all of the weight
    # decay terms (L2 loss).
    return tf.add_n(tf.get_collection('losses'), name='total_loss') 

def language_model_graph(self, compute_loss=True):
    """Constructs LM graph from inputs to LM loss.

    * Caches the VatxtInput object in `self.lm_inputs`
    * Caches tensors: `lm_embedded`

    Args:
      compute_loss: bool, whether to compute and return the loss or stop after
        the LSTM computation.

    Returns:
      loss: scalar float.
    """
    inputs = _inputs('train', pretrain=True)
    self.lm_inputs = inputs
    return self._lm_loss(inputs, compute_loss=compute_loss) 

def language_model_graph(self, compute_loss=True):
    """Constructs LM graph from inputs to LM loss.

    * Caches the VatxtInput object in `self.lm_inputs`
    * Caches tensors: `lm_embedded`

    Args:
      compute_loss: bool, whether to compute and return the loss or stop after
        the LSTM computation.

    Returns:
      loss: scalar float.
    """
    inputs = _inputs('train', pretrain=True)
    self.lm_inputs = inputs
    return self._lm_loss(inputs, compute_loss=compute_loss) 

def _lm_loss(self,
               inputs,
               emb_key='lm_embedded',
               lstm_layer='lstm',
               lm_loss_layer='lm_loss',
               loss_name='lm_loss',
               compute_loss=True):
    embedded = self.layers['embedding'](inputs.tokens)
    self.tensors[emb_key] = embedded
    lstm_out, next_state = self.layers[lstm_layer](embedded, inputs.state,
                                                   inputs.length)
    if compute_loss:
      loss = self.layers[lm_loss_layer](
          [lstm_out, inputs.labels, inputs.weights])
      with tf.control_dependencies([inputs.save_state(next_state)]):
        loss = tf.identity(loss)
        tf.summary.scalar(loss_name, loss)

      return loss 

def _lm_loss(self,
               inputs,
               emb_key='lm_embedded',
               lstm_layer='lstm',
               lm_loss_layer='lm_loss',
               loss_name='lm_loss',
               compute_loss=True):
    embedded = self.layers['embedding'](inputs.tokens)
    self.tensors[emb_key] = embedded
    lstm_out, next_state = self.layers[lstm_layer](embedded, inputs.state,
                                                   inputs.length)
    if compute_loss:
      loss = self.layers[lm_loss_layer](
          [lstm_out, inputs.labels, inputs.weights])
      with tf.control_dependencies([inputs.save_state(next_state)]):
        loss = tf.identity(loss)
        tf.summary.scalar(loss_name, loss)

      return loss 

def classifier_graph(self):
    """Constructs classifier graph from inputs to classifier loss.

    * Caches the VatxtInput object in `self.cl_inputs`
    * Caches tensors: `cl_embedded`, `cl_logits`, `cl_loss`

    Returns:
      loss: scalar float.
    """
    inputs = _inputs('train', pretrain=False)
    self.cl_inputs = inputs
    embedded = self.layers['embedding'](inputs.tokens)
    self.tensors['cl_embedded'] = embedded

    _, next_state, logits, loss = self.cl_loss_from_embedding(
        embedded, return_intermediates=True)
    tf.summary.scalar('classification_loss', loss)
    self.tensors['cl_logits'] = logits
    self.tensors['cl_loss'] = loss

    acc = layers_lib.accuracy(logits, inputs.labels, inputs.weights)
    tf.summary.scalar('accuracy', acc)

    adv_loss = (self.adversarial_loss() * tf.constant(
        FLAGS.adv_reg_coeff, name='adv_reg_coeff'))
    tf.summary.scalar('adversarial_loss', adv_loss)

    total_loss = loss + adv_loss
    tf.summary.scalar('total_classification_loss', total_loss)

    with tf.control_dependencies([inputs.save_state(next_state)]):
      total_loss = tf.identity(total_loss)

    return total_loss 

def language_model_graph(self, compute_loss=True):
    """Constructs forward and reverse LM graphs from inputs to LM losses.

    * Caches the VatxtInput objects in `self.lm_inputs`
    * Caches tensors: `lm_embedded`, `lm_embedded_reverse`

    Args:
      compute_loss: bool, whether to compute and return the loss or stop after
        the LSTM computation.

    Returns:
      loss: scalar float, sum of forward and reverse losses.
    """
    inputs = _inputs('train', pretrain=True, bidir=True)
    self.lm_inputs = inputs
    f_inputs, r_inputs = inputs
    f_loss = self._lm_loss(f_inputs, compute_loss=compute_loss)
    r_loss = self._lm_loss(
        r_inputs,
        emb_key='lm_embedded_reverse',
        lstm_layer='lstm_reverse',
        lm_loss_layer='lm_loss_reverse',
        loss_name='lm_loss_reverse',
        compute_loss=compute_loss)
    if compute_loss:
      return f_loss + r_loss 

def cl_loss_from_embedding(self,
                             embedded,
                             inputs=None,
                             return_intermediates=False):
    """Compute classification loss from embedding.

    Args:
      embedded: 3-D float Tensor [batch_size, num_timesteps, embedding_dim]
      inputs: VatxtInput, defaults to self.cl_inputs.
      return_intermediates: bool, whether to return intermediate tensors or only
        the final loss.

    Returns:
      If return_intermediates is True:
        lstm_out, next_state, logits, loss
      Else:
        loss
    """
    if inputs is None:
      inputs = self.cl_inputs

    lstm_out, next_state = self.layers['lstm'](embedded, inputs.state,
                                               inputs.length)
    logits = self.layers['cl_logits'](lstm_out)
    loss = layers_lib.classification_loss(logits, inputs.labels, inputs.weights)

    if return_intermediates:
      return lstm_out, next_state, logits, loss
    else:
      return loss 

def eval_graph(self, dataset='test'):
    """Constructs classifier evaluation graph.

    Args:
      dataset: the labeled dataset to evaluate, {'train', 'test', 'valid'}.

    Returns:
      eval_ops: dict<metric name, tuple(value, update_op)>
      var_restore_dict: dict mapping variable restoration names to variables.
        Trainable variables will be mapped to their moving average names.
    """
    inputs = _inputs(dataset, pretrain=False)
    embedded = self.layers['embedding'](inputs.tokens)
    _, next_state, logits, _ = self.cl_loss_from_embedding(
        embedded, inputs=inputs, return_intermediates=True)

    eval_ops = {
        'accuracy':
            tf.contrib.metrics.streaming_accuracy(
                layers_lib.predictions(logits), inputs.labels, inputs.weights)
    }

    with tf.control_dependencies([inputs.save_state(next_state)]):
      acc, acc_update = eval_ops['accuracy']
      acc_update = tf.identity(acc_update)
      eval_ops['accuracy'] = (acc, acc_update)

    var_restore_dict = make_restore_average_vars_dict()
    return eval_ops, var_restore_dict 

def cl_loss_from_embedding(self,
                             embedded,
                             inputs=None,
                             return_intermediates=False):
    """Compute classification loss from embedding.

    Args:
      embedded: Length 2 tuple of 3-D float Tensor
        [batch_size, num_timesteps, embedding_dim].
      inputs: Length 2 tuple of VatxtInput, defaults to self.cl_inputs.
      return_intermediates: bool, whether to return intermediate tensors or only
        the final loss.

    Returns:
      If return_intermediates is True:
        lstm_out, next_states, logits, loss
      Else:
        loss
    """
    if inputs is None:
      inputs = self.cl_inputs

    out = []
    for (layer_name, emb, inp) in zip(['lstm', 'lstm_reverse'], embedded,
                                      inputs):
      out.append(self.layers[layer_name](emb, inp.state, inp.length))
    lstm_outs, next_states = zip(*out)

    # Concatenate output of forward and reverse LSTMs
    lstm_out = tf.concat(lstm_outs, 1)

    logits = self.layers['cl_logits'](lstm_out)
    f_inputs, _ = inputs  # pylint: disable=unpacking-non-sequence
    loss = layers_lib.classification_loss(logits, f_inputs.labels,
                                          f_inputs.weights)

    if return_intermediates:
      return lstm_out, next_states, logits, loss
    else:
      return loss 

def eval_graph(self, dataset='test'):
    """Constructs classifier evaluation graph.

    Args:
      dataset: the labeled dataset to evaluate, {'train', 'test', 'valid'}.

    Returns:
      eval_ops: dict<metric name, tuple(value, update_op)>
      var_restore_dict: dict mapping variable restoration names to variables.
        Trainable variables will be mapped to their moving average names.
    """
    inputs = _inputs(dataset, pretrain=False, bidir=True)
    embedded = [self.layers['embedding'](inp.tokens) for inp in inputs]
    _, next_states, logits, _ = self.cl_loss_from_embedding(
        embedded, inputs=inputs, return_intermediates=True)
    f_inputs, _ = inputs

    eval_ops = {
        'accuracy':
            tf.contrib.metrics.streaming_accuracy(
                layers_lib.predictions(logits), f_inputs.labels,
                f_inputs.weights)
    }

    # Save states on accuracy update
    saves = [inp.save_state(state) for (inp, state) in zip(inputs, next_states)]
    with tf.control_dependencies(saves):
      acc, acc_update = eval_ops['accuracy']
      acc_update = tf.identity(acc_update)
      eval_ops['accuracy'] = (acc, acc_update)

    var_restore_dict = make_restore_average_vars_dict()
    return eval_ops, var_restore_dict 

def cl_loss_from_embedding(self,
                             embedded,
                             inputs=None,
                             return_intermediates=False):
    """Compute classification loss from embedding.

    Args:
      embedded: Length 2 tuple of 3-D float Tensor
        [batch_size, num_timesteps, embedding_dim].
      inputs: Length 2 tuple of VatxtInput, defaults to self.cl_inputs.
      return_intermediates: bool, whether to return intermediate tensors or only
        the final loss.

    Returns:
      If return_intermediates is True:
        lstm_out, next_states, logits, loss
      Else:
        loss
    """
    if inputs is None:
      inputs = self.cl_inputs

    out = []
    for (layer_name, emb, inp) in zip(['lstm', 'lstm_reverse'], embedded,
                                      inputs):
      out.append(self.layers[layer_name](emb, inp.state, inp.length))
    lstm_outs, next_states = zip(*out)

    # Concatenate output of forward and reverse LSTMs
    lstm_out = tf.concat(lstm_outs, 1)

    logits = self.layers['cl_logits'](lstm_out)
    f_inputs, _ = inputs  # pylint: disable=unpacking-non-sequence
    loss = layers_lib.classification_loss(logits, f_inputs.labels,
                                          f_inputs.weights)

    if return_intermediates:
      return lstm_out, next_states, logits, loss
    else:
      return loss 

def classifier_graph(self):
    """Constructs classifier graph from inputs to classifier loss.

    * Caches the VatxtInput object in `self.cl_inputs`
    * Caches tensors: `cl_embedded`, `cl_logits`, `cl_loss`

    Returns:
      loss: scalar float.
    """
    inputs = _inputs('train', pretrain=False)
    self.cl_inputs = inputs
    embedded = self.layers['embedding'](inputs.tokens)
    self.tensors['cl_embedded'] = embedded

    _, next_state, logits, loss = self.cl_loss_from_embedding(
        embedded, return_intermediates=True)
    tf.summary.scalar('classification_loss', loss)
    self.tensors['cl_logits'] = logits
    self.tensors['cl_loss'] = loss

    acc = layers_lib.accuracy(logits, inputs.labels, inputs.weights)
    tf.summary.scalar('accuracy', acc)

    adv_loss = (self.adversarial_loss() * tf.constant(
        FLAGS.adv_reg_coeff, name='adv_reg_coeff'))
    tf.summary.scalar('adversarial_loss', adv_loss)

    total_loss = loss + adv_loss
    tf.summary.scalar('total_classification_loss', total_loss)

    with tf.control_dependencies([inputs.save_state(next_state)]):
      total_loss = tf.identity(total_loss)

    return total_loss 

def eval_graph(self, dataset='test'):
    """Constructs classifier evaluation graph.

    Args:
      dataset: the labeled dataset to evaluate, {'train', 'test', 'valid'}.

    Returns:
      eval_ops: dict<metric name, tuple(value, update_op)>
      var_restore_dict: dict mapping variable restoration names to variables.
        Trainable variables will be mapped to their moving average names.
    """
    inputs = _inputs(dataset, pretrain=False, bidir=True)
    embedded = [self.layers['embedding'](inp.tokens) for inp in inputs]
    _, next_states, logits, _ = self.cl_loss_from_embedding(
        embedded, inputs=inputs, return_intermediates=True)
    f_inputs, _ = inputs

    eval_ops = {
        'accuracy':
            tf.contrib.metrics.streaming_accuracy(
                layers_lib.predictions(logits), f_inputs.labels,
                f_inputs.weights)
    }

    # Save states on accuracy update
    saves = [inp.save_state(state) for (inp, state) in zip(inputs, next_states)]
    with tf.control_dependencies(saves):
      acc, acc_update = eval_ops['accuracy']
      acc_update = tf.identity(acc_update)
      eval_ops['accuracy'] = (acc, acc_update)

    var_restore_dict = make_restore_average_vars_dict()
    return eval_ops, var_restore_dict 

def language_model_graph(self, compute_loss=True):
    """Constructs forward and reverse LM graphs from inputs to LM losses.

    * Caches the VatxtInput objects in `self.lm_inputs`
    * Caches tensors: `lm_embedded`, `lm_embedded_reverse`

    Args:
      compute_loss: bool, whether to compute and return the loss or stop after
        the LSTM computation.

    Returns:
      loss: scalar float, sum of forward and reverse losses.
    """
    inputs = _inputs('train', pretrain=True, bidir=True)
    self.lm_inputs = inputs
    f_inputs, r_inputs = inputs
    f_loss = self._lm_loss(f_inputs, compute_loss=compute_loss)
    r_loss = self._lm_loss(
        r_inputs,
        emb_key='lm_embedded_reverse',
        lstm_layer='lstm_reverse',
        lm_loss_layer='lm_loss_reverse',
        loss_name='lm_loss_reverse',
        compute_loss=compute_loss)
    if compute_loss:
      return f_loss + r_loss 

def language_model_graph(self, compute_loss=True):
    """Constructs forward and reverse LM graphs from inputs to LM losses.

    * Caches the VatxtInput objects in `self.lm_inputs`
    * Caches tensors: `lm_embedded`, `lm_embedded_reverse`

    Args:
      compute_loss: bool, whether to compute and return the loss or stop after
        the LSTM computation.

    Returns:
      loss: scalar float, sum of forward and reverse losses.
    """
    inputs = _inputs('train', pretrain=True, bidir=True)
    self.lm_inputs = inputs
    f_inputs, r_inputs = inputs
    f_loss = self._lm_loss(f_inputs, compute_loss=compute_loss)
    r_loss = self._lm_loss(
        r_inputs,
        emb_key='lm_embedded_reverse',
        lstm_layer='lstm_reverse',
        lm_loss_layer='lm_loss_reverse',
        loss_name='lm_loss_reverse',
        compute_loss=compute_loss)
    if compute_loss:
      return f_loss + r_loss 

def cl_loss_from_embedding(self,
                             embedded,
                             inputs=None,
                             return_intermediates=False):
    """Compute classification loss from embedding.

    Args:
      embedded: 3-D float Tensor [batch_size, num_timesteps, embedding_dim]
      inputs: VatxtInput, defaults to self.cl_inputs.
      return_intermediates: bool, whether to return intermediate tensors or only
        the final loss.

    Returns:
      If return_intermediates is True:
        lstm_out, next_state, logits, loss
      Else:
        loss
    """
    if inputs is None:
      inputs = self.cl_inputs

    lstm_out, next_state = self.layers['lstm'](embedded, inputs.state,
                                               inputs.length)
    if FLAGS.single_label:
      indices = tf.stack([tf.range(FLAGS.batch_size), inputs.length - 1], 1)
      lstm_out = tf.expand_dims(tf.gather_nd(lstm_out, indices), 1)
      labels = tf.expand_dims(tf.gather_nd(inputs.labels, indices), 1)
      weights = tf.expand_dims(tf.gather_nd(inputs.weights, indices), 1)
    else:
      labels = inputs.labels
      weights = inputs.weights
    logits = self.layers['cl_logits'](lstm_out)
    loss = layers_lib.classification_loss(logits, labels, weights)

    if return_intermediates:
      return lstm_out, next_state, logits, loss
    else:
      return loss