Python tensorflow.contrib.rnn.GRUCell() Examples

def __init__(self,
                 state_size,
                 embed_size,
                 dropout_prob,
                 num_layers,
                 base_cell="GRUCell",
                 state_wrapper=None):
        """
        Args:
            state_size: number of units in underlying rnn cell.
            embed_size: dimension size of word-embedding space.
            dropout_prob: probability of a node being dropped.
            num_layers: how many cells to include in the MultiRNNCell.
            base_cell: (str) name of underling cell to use (e.g. 'GRUCell')
            state_wrapper: allow states to store their wrapper class. See the
                wrapper method docstring below for more info.
        """
        self.state_size = state_size
        self.embed_size = embed_size
        self.num_layers = num_layers
        self.dropout_prob = dropout_prob
        self.base_cell = base_cell
        self._wrapper = state_wrapper 

def answer_module(self):
        """ Answer Module:generate an answer from the final memory vector.
        Input:
            hidden state from episodic memory module:[batch_size,hidden_size]
            question:[batch_size, embedding_size]
        """
        steps=self.sequence_length if self.decode_with_sequences else 1 #decoder for a list of tokens with sequence. e.g."x1 x2 x3 x4..."
        a=self.m_T #init hidden state
        y_pred=tf.zeros((self.batch_size,self.hidden_size)) #TODO usually we will init this as a special token '<GO>', you can change this line by pass embedding of '<GO>' from outside.
        logits_list=[]
        logits_return=None
        for i in range(steps):
            cell = rnn.GRUCell(self.hidden_size)
            y_previous_q=tf.concat([y_pred,self.query_embedding],axis=1) #[batch_hidden_size*2]
            _, a = cell( y_previous_q,a)
            logits=tf.layers.dense(a,units=self.num_classes) #[batch_size,vocab_size]
            logits_list.append(logits)
        if self.decode_with_sequences:#need to get sequences.
            logits_return = tf.stack(logits_list, axis=1)  # [batch_size,sequence_length,num_classes]
        else:#only need to get an answer, not sequences
            logits_return = logits_list[0]  #[batcj_size,num_classes]

        return logits_return 

def __init__(self,
               cell,
               attention_mechanism,
               attention_layer_size=None,
               alignment_history=False,
               cell_input_fn=None,
               output_attention=True,
               initial_cell_state=None,
               name=None):
    if not isinstance(cell, (rnn.LSTMCell, rnn.GRUCell)):
      raise ValueError('SyncAttentionWrapper only supports LSTMCell and GRUCell, '
                       'Got: {}'.format(cell))
    super(SyncAttentionWrapper, self).__init__(
      cell,
      attention_mechanism,
      attention_layer_size=attention_layer_size,
      alignment_history=alignment_history,
      cell_input_fn=cell_input_fn,
      output_attention=output_attention,
      initial_cell_state=initial_cell_state,
      name=name
    ) 

def bidirectional_RNN(self, num_hidden, inputs):
    """
    desc: create bidirectional rnn layer
    args:
      num_hidden: number of hidden units
      inputs: input word or sentence
    returns:
      concatenated encoder and decoder outputs
    """

    with tf.name_scope("bidirectional_RNN"):
      encoder_fw_cell = rnn.GRUCell(num_hidden)
      encoder_bw_cell = rnn.GRUCell(num_hidden)
      ((encoder_fw_outputs, encoder_bw_outputs), (_, _)) = tf.nn.bidirectional_dynamic_rnn(cell_fw=encoder_fw_cell,
                                                                                           cell_bw=encoder_bw_cell,
                                                                                           inputs=inputs,
                                                                                           dtype=tf.float32,
                                                                                           time_major=True)
      encoder_outputs = tf.concat((encoder_fw_outputs, encoder_bw_outputs), 2)
      return encoder_outputs
  # end 

def __init__(self, ob_space, ac_space, size=256, **kwargs):
        self.x = x = tf.placeholder(tf.float32, [None] + list(ob_space))

        for i in range(4):
            x = tf.nn.elu(conv2d(x, 32, "l{}".format(i + 1), [3, 3], [2, 2]))
        # introduce a "fake" batch dimension of 1 after flatten so that we can do GRU over time dim
        x = tf.expand_dims(flatten(x), 1)

        gru = rnn.GRUCell(size)

        h_init = np.zeros((1, size), np.float32)
        self.state_init = [h_init]
        h_in = tf.placeholder(tf.float32, [1, size])
        self.state_in = [h_in]

        gru_outputs, gru_state = tf.nn.dynamic_rnn(
            gru, x, initial_state=h_in, sequence_length=[size], time_major=True)
        x = tf.reshape(gru_outputs, [-1, size])
        self.logits = linear(x, ac_space, "action", normalized_columns_initializer(0.01))
        self.vf = tf.reshape(linear(x, 1, "value", normalized_columns_initializer(1.0)), [-1])
        self.state_out = [gru_state[:1]]
        self.sample = categorical_sample(self.logits, ac_space)[0, :]
        self.var_list = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, tf.get_variable_scope().name) 

def _create_rnn_cell(self):
        """
        Creates a single RNN cell according to the architecture of this RNN.
        
        Returns
        -------
        rnn cell
            A single RNN cell according to the architecture of this RNN
        """
        keep_prob = 1.0 if self.keep_prob is None else self.keep_prob

        if self.cell_type == CellType.GRU:
            return DropoutWrapper(GRUCell(self.num_units), keep_prob, keep_prob)
        elif self.cell_type == CellType.LSTM:
            return DropoutWrapper(LSTMCell(self.num_units), keep_prob, keep_prob)
        else:
            raise ValueError("unknown cell type: {}".format(self.cell_type)) 

def __init__(self, ob_space, ac_space, size=256, **kwargs):
        self.x = x = tf.placeholder(tf.float32, [None] + list(ob_space))

        for i in range(4):
            x = tf.nn.elu(conv2d(x, 32, "l{}".format(i + 1), [3, 3], [2, 2]))
        # introduce a "fake" batch dimension of 1 after flatten so that we can do GRU over time dim
        x = tf.expand_dims(flatten(x), 1)

        gru = rnn.GRUCell(size)

        h_init = np.zeros((1, size), np.float32)
        self.state_init = [h_init]
        h_in = tf.placeholder(tf.float32, [1, size])
        self.state_in = [h_in]

        gru_outputs, gru_state = tf.nn.dynamic_rnn(
            gru, x, initial_state=h_in, sequence_length=[size], time_major=True)
        x = tf.reshape(gru_outputs, [-1, size])
        self.logits = linear(x, ac_space, "action", normalized_columns_initializer(0.01))
        self.vf = tf.reshape(linear(x, 1, "value", normalized_columns_initializer(1.0)), [-1])
        self.state_out = [gru_state[:1]]
        self.sample = categorical_sample(self.logits, ac_space)[0, :]
        self.var_list = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, tf.get_variable_scope().name) 

def __init__(self,
               cell,
               attention_mechanism,
               attention_layer_size=None,
               alignment_history=False,
               cell_input_fn=None,
               output_attention=True,
               initial_cell_state=None,
               name=None):
    if not isinstance(cell, (rnn.LSTMCell, rnn.GRUCell)):
      raise ValueError('SyncAttentionWrapper only supports LSTMCell and GRUCell, '
                       'Got: {}'.format(cell))
    super(SyncAttentionWrapper, self).__init__(
      cell,
      attention_mechanism,
      attention_layer_size=attention_layer_size,
      alignment_history=alignment_history,
      cell_input_fn=cell_input_fn,
      output_attention=output_attention,
      initial_cell_state=initial_cell_state,
      name=name
    ) 

def __init__(self, state_size, num_layers, dropout_prob, base_cell):
        """Define the cell by composing/wrapping with tf.contrib.rnn functions.
        
        Args:
            state_size: number of units in the cell.
            num_layers: how many cells to include in the MultiRNNCell.
            dropout_prob: probability of a node being dropped.
            base_cell: (str) name of underling cell to use (e.g. 'GRUCell')
        """

        self._state_size = state_size
        self._num_layers = num_layers
        self._dropout_prob = dropout_prob
        self._base_cell = base_cell

        def single_cell():
            """Convert cell name (str) to class, and create it."""
            return getattr(tf.contrib.rnn, base_cell)(num_units=state_size)

        if num_layers == 1:
            self._cell = single_cell()
        else:
            self._cell = MultiRNNCell(
                [single_cell() for _ in range(num_layers)]) 

def ReferenceEncoder(inputs, input_lengths, filters, kernel_size, strides, is_training, scope='reference_encoder'):
    with tf.variable_scope(scope):
        reference_output = tf.expand_dims(inputs, axis=-1)
        for i, channel in enumerate(filters):
            reference_output = conv2d(reference_output, channel, kernel_size,
                      strides, tf.nn.relu, is_training, 'conv2d_{}'.format(i))

        shape = shape_list(reference_output)
        reference_output = tf.reshape(reference_output, shape[:-2] + [shape[2] * shape[3]])

        #GRU
        encoder_outputs, encoder_state = tf.nn.dynamic_rnn(
           cell=GRUCell(128),
           inputs=reference_output,
           sequence_length=input_lengths,
           dtype=tf.float32
        )
        return encoder_state 

def answer_module(self):
        """ Answer Module:generate an answer from the final memory vector.
        Input:
            hidden state from episodic memory module:[batch_size,hidden_size]
            question:[batch_size, embedding_size]
        """
        steps=self.sequence_length if self.decode_with_sequences else 1 #decoder for a list of tokens with sequence. e.g."x1 x2 x3 x4..."
        a=self.m_T #init hidden state
        y_pred=tf.zeros((self.batch_size,self.hidden_size)) #TODO usually we will init this as a special token '<GO>', you can change this line by pass embedding of '<GO>' from outside.
        logits_list=[]
        logits_return=None
        for i in range(steps):
            cell = rnn.GRUCell(self.hidden_size)
            y_previous_q=tf.concat([y_pred,self.query_embedding],axis=1) #[batch_hidden_size*2]
            _, a = cell( y_previous_q,a)
            logits=tf.layers.dense(a,units=self.num_classes) #[batch_size,vocab_size]
            logits_list.append(logits)
        if self.decode_with_sequences:#need to get sequences.
            logits_return = tf.stack(logits_list, axis=1)  # [batch_size,sequence_length,num_classes]
        else:#only need to get an answer, not sequences
            logits_return = logits_list[0]  #[batcj_size,num_classes]

        return logits_return 

def biLSTM_layer(self):
        with tf.variable_scope("bi-LSTM") as scope:
            lstm_cell = {}
            for direction in ["forward", "backward"]:
                with tf.variable_scope(direction):
                    lstm_cell[direction] = rnn.GRUCell(
                        num_units=self.lstm_dim,
                        # use_peepholes=True,
                        # initializer=self.initializer,
                        # state_is_tuple=True
                    )

            outputs, final_states = tf.nn.bidirectional_dynamic_rnn(
                cell_fw=lstm_cell['forward'],
                cell_bw=lstm_cell['backward'],
                inputs=self.model_inputs,
                sequence_length=self.length,
                dtype=tf.float32,
            )
            self.lstm_outputs = tf.concat(outputs, axis=2) 

def __init__(self,
               num_units,
               tied=False,
               non_recurrent_fn=None,
               state_is_tuple=True,
               output_is_tuple=True):
    super(Grid2GRUCell, self).__init__(
        num_units=num_units,
        num_dims=2,
        input_dims=0,
        output_dims=0,
        priority_dims=0,
        tied=tied,
        non_recurrent_dims=None if non_recurrent_fn is None else 0,
        cell_fn=lambda n: rnn.GRUCell(num_units=n),
        non_recurrent_fn=non_recurrent_fn,
        state_is_tuple=state_is_tuple,
        output_is_tuple=output_is_tuple)


# Helpers 

def ReferenceEncoder(inputs, input_lengths, filters, kernel_size, strides, is_training, scope='reference_encoder'):
    with tf.variable_scope(scope):
        reference_output = tf.expand_dims(inputs, axis=-1)
        for i, channel in enumerate(filters):
            reference_output = conv2d(reference_output, channel, kernel_size,
                      strides, tf.nn.relu, is_training, 'conv2d_{}'.format(i))

        shape = shape_list(reference_output)
        reference_output = tf.reshape(reference_output, shape[:-2] + [shape[2] * shape[3]])

        #GRU
        encoder_outputs, encoder_state = tf.nn.dynamic_rnn(
           cell=GRUCell(128),
           inputs=reference_output,
           sequence_length=input_lengths,
           dtype=tf.float32
        )
        return encoder_state 

def build_cell(self, name=None):
        if self.hparams.cell_type == 'linear':
            cell = BasicRNNCell(self.hparams.hidden_units,
                                activation=tf.identity, name=name)
        elif self.hparams.cell_type == 'tanh':
            cell = BasicRNNCell(self.hparams.hidden_units,
                                activation=tf.tanh, name=name)
        elif self.hparams.cell_type == 'relu':
            cell = BasicRNNCell(self.hparams.hidden_units,
                                activation=tf.nn.relu, name=name)
        elif self.hparams.cell_type == 'gru':
            cell = GRUCell(self.hparams.hidden_units, name=name)
        elif self.hparams.cell_type == 'lstm':
            cell = LSTMCell(self.hparams.hidden_units, name=name, state_is_tuple=False)
        else:
            raise ValueError('Provided cell type not supported.')
        return cell 

def build_cell(units, cell_type='lstm', num_layers=1):
	if num_layers > 1:
		cell = rnn.MultiRNNCell([
			build_cell(units, cell_type, 1) for _ in range(num_layers)
		])
	else:
		if cell_type == "lstm":
			cell = rnn.LSTMCell(units)
		elif cell_type == "gru":
			cell = rnn.GRUCell(units)
		else:
			raise ValueError('Do not support %s' % cell_type)
	return cell 

def __init__(self, num_units, tied=False, non_recurrent_fn=None):
    super(Grid2GRUCell, self).__init__(
        num_units=num_units, num_dims=2,
        input_dims=0, output_dims=0, priority_dims=0, tied=tied,
        non_recurrent_dims=None if non_recurrent_fn is None else 0,
        cell_fn=lambda n, i: rnn.GRUCell(num_units=n, input_size=i),
        non_recurrent_fn=non_recurrent_fn) 

def _single_cell(unit_type,
                 num_units,
                 forget_bias,
                 dropout,
                 mode,
                 residual_connection=False,
                 residual_fn=None,
                 trainable=True):
  """Create an instance of a single RNN cell."""
  # dropout (= 1 - keep_prob) is set to 0 during eval and infer
  dropout = dropout if mode == tf.estimator.ModeKeys.TRAIN else 0.0

  # Cell Type
  if unit_type == "lstm":
    single_cell = contrib_rnn.LSTMCell(
        num_units, forget_bias=forget_bias, trainable=trainable)
  elif unit_type == "gru":
    single_cell = contrib_rnn.GRUCell(num_units, trainable=trainable)
  elif unit_type == "layer_norm_lstm":
    single_cell = contrib_rnn.LayerNormBasicLSTMCell(
        num_units,
        forget_bias=forget_bias,
        layer_norm=True,
        trainable=trainable)
  elif unit_type == "nas":
    single_cell = contrib_rnn.NASCell(num_units, trainable=trainable)
  else:
    raise ValueError("Unknown unit type %s!" % unit_type)

  # Dropout (= 1 - keep_prob).
  if dropout > 0.0:
    single_cell = contrib_rnn.DropoutWrapper(
        cell=single_cell, input_keep_prob=(1.0 - dropout))

  # Residual.
  if residual_connection:
    single_cell = contrib_rnn.ResidualWrapper(
        single_cell, residual_fn=residual_fn)

  return single_cell 

def gru_cell(self):
        with tf.name_scope('gru_cell'):
            # activation：tanh
            cell = rnn.GRUCell(self.hidden_size, reuse=tf.get_variable_scope().reuse)
        return rnn.DropoutWrapper(cell, output_keep_prob=self.keep_prob) 

def __init__(self, m, seq_len, name='gen', reuse=False, n_stack=1,
        logit_range=4.0, **kwargs):
        # Get GRU cell builder
        range_wrapper = partial(OutputRangeWrapper, output_range=logit_range)
        cb = GeneratorRNNCellBuilder(
            rnn.GRUCell, m=m, n_stack=n_stack, wrappers=[range_wrapper]
        )
        # Super constructor
        super(GRUGenerator, self).__init__(
            m, seq_len, name=name, cell_builder=cb, reuse=reuse, **kwargs
        ) 

def input_module(self):
        """encode raw texts into vector representation"""
        story_embedding=tf.nn.embedding_lookup(self.Embedding,self.story)  # [batch_size,story_length,sequence_length,embed_size]
        story_embedding=tf.reshape(story_embedding,(self.batch_size,self.story_length,self.sequence_length*self.embed_size))
        hidden_state=tf.ones((self.batch_size,self.hidden_size),dtype=tf.float32)
        cell = rnn.GRUCell(self.hidden_size)
        self.story_embedding,hidden_state=tf.nn.dynamic_rnn(cell,story_embedding,dtype=tf.float32,scope="input_module") 

def _init_word_encoder(self):
    with tf.variable_scope('word') as scope:
      word_rnn_inputs = tf.reshape(
        self.embedded_inputs,
        [-1, self.max_num_words, self.emb_size]
      )
      sentence_lengths = tf.reshape(self.sentence_lengths, [-1])

      # word encoder
      cell_fw = rnn.GRUCell(self.hidden_dim)
      cell_bw = rnn.GRUCell(self.hidden_dim)

      init_state_fw = tf.tile(tf.get_variable('init_state_fw',
                                              shape=[1, self.hidden_dim],
                                              initializer=tf.constant_initializer(1.0)),
                              multiples=[get_shape(word_rnn_inputs)[0], 1])
      init_state_bw = tf.tile(tf.get_variable('init_state_bw',
                                              shape=[1, self.hidden_dim],
                                              initializer=tf.constant_initializer(1.0)),
                              multiples=[get_shape(word_rnn_inputs)[0], 1])

      word_rnn_outputs, _ = bidirectional_rnn(
        cell_fw=cell_fw,
        cell_bw=cell_bw,
        inputs=word_rnn_inputs,
        input_lengths=sentence_lengths,
        initial_state_fw=init_state_fw,
        initial_state_bw=init_state_bw,
        scope=scope
      )

      self.word_outputs, self.word_att_weights = text_attention(inputs=word_rnn_outputs,
                                                                att_dim=self.att_dim,
                                                                sequence_lengths=sentence_lengths)

      self.word_outputs = tf.nn.dropout(self.word_outputs, keep_prob=self.dropout_keep_prob) 

def _init_sent_encoder(self):
    with tf.variable_scope('sentence') as scope:
      sentence_rnn_inputs = tf.reshape(self.word_outputs, [-1, self.max_num_sents, 2 * self.hidden_dim])

      # sentence encoder
      cell_fw = rnn.GRUCell(self.hidden_dim)
      cell_bw = rnn.GRUCell(self.hidden_dim)

      init_state_fw = tf.tile(tf.get_variable('init_state_fw',
                                              shape=[1, self.hidden_dim],
                                              initializer=tf.constant_initializer(1.0)),
                              multiples=[get_shape(sentence_rnn_inputs)[0], 1])
      init_state_bw = tf.tile(tf.get_variable('init_state_bw',
                                              shape=[1, self.hidden_dim],
                                              initializer=tf.constant_initializer(1.0)),
                              multiples=[get_shape(sentence_rnn_inputs)[0], 1])

      sentence_rnn_outputs, _ = bidirectional_rnn(
        cell_fw=cell_fw,
        cell_bw=cell_bw,
        inputs=sentence_rnn_inputs,
        input_lengths=self.document_lengths,
        initial_state_fw=init_state_fw,
        initial_state_bw=init_state_bw,
        scope=scope
      )

      self.sentence_outputs, self.sent_att_weights, self.img_att_weights = visual_aspect_attention(
          text_input=sentence_rnn_outputs,
          visual_input=self.images,
          att_dim=self.att_dim,
          sequence_lengths=self.document_lengths
        )

      self.sentence_outputs = tf.nn.dropout(self.sentence_outputs, keep_prob=self.dropout_keep_prob) 

def _inference(self):
        logging.info('...create inference')

        fw_state_tuple = self.unstack_fw_states(self.fw_state)

        fw_cells   = list()
        for i in range(0, self.num_layers):
            if (self.cell_type == 'lstm'):
                cell = rnn.LSTMCell(num_units=self.cell_sizes[i], state_is_tuple=True)
            elif (self.cell_type == 'gru'):
                # change to GRU
                cell = rnn.GRUCell(num_units=self.cell_sizes[i])
            else:
                cell = rnn.BasicRNNCell(num_units=self.cell_sizes[i])

            cell = rnn.DropoutWrapper(cell, output_keep_prob=self.keep_prob)
            fw_cells.append(cell)
        self.fw_cells = rnn.MultiRNNCell(fw_cells, state_is_tuple=True)

        rnn_outputs, states = tf.nn.dynamic_rnn(
            self.fw_cells, 
            self.inputs, 
            initial_state=fw_state_tuple,
            sequence_length=self.seq_lengths,
            dtype=tf.float32, time_major=True)

        # project output from rnn output size to OUTPUT_SIZE. Sometimes it is worth adding
        # an extra layer here.
        self.projection = lambda x: layers.linear(x, 
            num_outputs=self.label_classes, activation_fn=tf.nn.sigmoid)

        self.logits = tf.map_fn(self.projection, rnn_outputs, name="logits")
        self.probs  = tf.nn.softmax(self.logits, name="probs")
        self.states = states

        tf.add_to_collection('probs',  self.probs) 

def _witch_cell(self):
        """
        RNN 类型
        :return: 
        """
        cell_tmp = None
        if self.cell_type == 'lstm':
            cell_tmp = rnn.BasicLSTMCell(self.hidden_unit)
        elif self.cell_type == 'gru':
            cell_tmp = rnn.GRUCell(self.hidden_unit)
        # 是否需要进行dropout
        if self.dropout_rate is not None:
            cell_tmp = rnn.DropoutWrapper(cell_tmp, output_keep_prob=self.dropout_rate)
        return cell_tmp 

def question_module(self):
        """
        input:tokens of query:[batch_size,sequence_length]
        :return: representation of question:[batch_size,hidden_size]
        """
        query_embedding = tf.nn.embedding_lookup(self.Embedding, self.query)  # [batch_size,sequence_length,embed_size]
        cell=rnn.GRUCell(self.hidden_size)
        _,self.query_embedding=tf.nn.dynamic_rnn(cell,query_embedding,dtype=tf.float32,scope="question_module") #query_embedding:[batch_size,hidden_size] 

def question_module(self):
        """
        input:tokens of query:[batch_size,sequence_length]
        :return: representation of question:[batch_size,hidden_size]
        """
        query_embedding = tf.nn.embedding_lookup(self.Embedding, self.query)  # [batch_size,sequence_length,embed_size]
        cell=rnn.GRUCell(self.hidden_size)
        _,self.query_embedding=tf.nn.dynamic_rnn(cell,query_embedding,dtype=tf.float32,scope="question_module") #query_embedding:[batch_size,hidden_size] 

def __init__(self, num_units, tied=False, non_recurrent_fn=None):
    super(Grid2GRUCell, self).__init__(
        num_units=num_units, num_dims=2,
        input_dims=0, output_dims=0, priority_dims=0, tied=tied,
        non_recurrent_dims=None if non_recurrent_fn is None else 0,
        cell_fn=lambda n, i: rnn.GRUCell(num_units=n, input_size=i),
        non_recurrent_fn=non_recurrent_fn) 

def __build(self):
        w_fc_in = self.__weight_variable([self.nClasses+1, 128], 'w_fc_in')
        b_fc_in = self.__bias_variable([128], 'b_fc_in')
        
        w_fc_o = self.__weight_variable([self.rnn_size, 128], 'w_fc_o')
        b_fc_o = self.__bias_variable([128], 'b_fc_o')
                
        w_output_action = self.__weight_variable([128, self.nClasses], 'w_fc_in')
        b_output_action = self.__bias_variable([self.nClasses], 'b_fc_in')
        
        w_output_len = self.__weight_variable([128, 2], 'w_fc_in')
        b_output_len = self.__bias_variable([2], 'b_fc_in')
        
        x = tf.reshape(self.input_seq, [-1, self.nClasses+1])
        h1 = tf.nn.relu(tf.matmul(x, w_fc_in) + b_fc_in)
        h1 = tf.reshape(h1, [-1,self.max_seq_sz,128])
        #rnn
        h1 = tf.unstack(h1, axis=1)
        def get_cell():
            return rnn.GRUCell(self.rnn_size)   
        gru_cell = rnn.MultiRNNCell([get_cell() for _ in range(self.num_layers)])
        outputs, states = rnn.static_rnn(gru_cell, h1, dtype=tf.float32) 
        #fc_o
        h2 = tf.nn.relu(tf.matmul(outputs[-1], w_fc_o) + b_fc_o)
        #output
        output_label = tf.matmul(h2, w_output_action) + b_output_action
        output_len = tf.nn.relu(tf.matmul(h2, w_output_len) + b_output_len)
        #    
        self.prediction = tf.concat([output_label, output_len], 1)
        self.saver = tf.train.Saver(write_version=tf.train.SaverDef.V2, max_to_keep=100) 

def _init_word_encoder(self):
    with tf.variable_scope('word-encoder') as scope:
      word_inputs = tf.reshape(self.embedded_inputs, [-1, self.max_word_length, self.emb_size])
      word_lengths = tf.reshape(self.word_lengths, [-1])

      # word encoder
      cell_fw = rnn.GRUCell(self.cell_dim, name='cell_fw')
      cell_bw = rnn.GRUCell(self.cell_dim, name='cell_bw')

      init_state_fw = tf.tile(tf.get_variable('init_state_fw',
                                              shape=[1, self.cell_dim],
                                              initializer=tf.constant_initializer(0)),
                              multiples=[get_shape(word_inputs)[0], 1])
      init_state_bw = tf.tile(tf.get_variable('init_state_bw',
                                              shape=[1, self.cell_dim],
                                              initializer=tf.constant_initializer(0)),
                              multiples=[get_shape(word_inputs)[0], 1])

      rnn_outputs, _ = bidirectional_rnn(cell_fw=cell_fw,
                                         cell_bw=cell_bw,
                                         inputs=word_inputs,
                                         input_lengths=word_lengths,
                                         initial_state_fw=init_state_fw,
                                         initial_state_bw=init_state_bw,
                                         scope=scope)

      word_outputs, word_att_weights = attention(inputs=rnn_outputs,
                                                 att_dim=self.att_dim,
                                                 sequence_lengths=word_lengths)
      self.word_outputs = tf.layers.dropout(word_outputs, self.dropout_rate, training=self.is_training)