Python tensorflow.get_variable_scope() Examples

def dense(x, size, name, weight_init=None, bias_init=0, weight_loss_dict=None, reuse=None):
    with tf.variable_scope(name, reuse=reuse):
        assert (len(tf.get_variable_scope().name.split('/')) == 2)

        w = tf.get_variable("w", [x.get_shape()[1], size], initializer=weight_init)
        b = tf.get_variable("b", [size], initializer=tf.constant_initializer(bias_init))
        weight_decay_fc = 3e-4

        if weight_loss_dict is not None:
            weight_decay = tf.multiply(tf.nn.l2_loss(w), weight_decay_fc, name='weight_decay_loss')
            if weight_loss_dict is not None:
                weight_loss_dict[w] = weight_decay_fc
                weight_loss_dict[b] = 0.0

            tf.add_to_collection(tf.get_variable_scope().name.split('/')[0] + '_' + 'losses', weight_decay)

        return tf.nn.bias_add(tf.matmul(x, w), b) 

def __init__(self, hparams=None):
        EncoderBase.__init__(self, hparams)
        use_bias = self._hparams.use_bias

        with tf.variable_scope(self.variable_scope):
            if self._hparams.initializer:
                tf.get_variable_scope().set_initializer(
                    layers.get_initializer(self._hparams.initializer))

            self.Q_dense = tf.layers.Dense(self._hparams.num_units,
                                           use_bias=use_bias,
                                           name='query')
            self.K_dense = tf.layers.Dense(self._hparams.num_units,
                                           use_bias=use_bias,
                                           name='key')
            self.V_dense = tf.layers.Dense(self._hparams.num_units,
                                           use_bias=use_bias,
                                           name='value')
            self.O_dense = tf.layers.Dense(self._hparams.output_dim,
                                           use_bias=use_bias,
                                           name='output') 

def testTrainEvalWithReuse(self):
    train_batch_size = 2
    eval_batch_size = 1
    train_height, train_width = 231, 231
    eval_height, eval_width = 281, 281
    num_classes = 1000
    with self.test_session():
      train_inputs = tf.random_uniform(
          (train_batch_size, train_height, train_width, 3))
      logits, _ = overfeat.overfeat(train_inputs)
      self.assertListEqual(logits.get_shape().as_list(),
                           [train_batch_size, num_classes])
      tf.get_variable_scope().reuse_variables()
      eval_inputs = tf.random_uniform(
          (eval_batch_size, eval_height, eval_width, 3))
      logits, _ = overfeat.overfeat(eval_inputs, is_training=False,
                                    spatial_squeeze=False)
      self.assertListEqual(logits.get_shape().as_list(),
                           [eval_batch_size, 2, 2, num_classes])
      logits = tf.reduce_mean(logits, [1, 2])
      predictions = tf.argmax(logits, 1)
      self.assertEquals(predictions.get_shape().as_list(), [eval_batch_size]) 

def testTrainEvalWithReuse(self):
    train_batch_size = 2
    eval_batch_size = 1
    train_height, train_width = 224, 224
    eval_height, eval_width = 256, 256
    num_classes = 1000
    with self.test_session():
      train_inputs = tf.random_uniform(
          (train_batch_size, train_height, train_width, 3))
      logits, _ = vgg.vgg_19(train_inputs)
      self.assertListEqual(logits.get_shape().as_list(),
                           [train_batch_size, num_classes])
      tf.get_variable_scope().reuse_variables()
      eval_inputs = tf.random_uniform(
          (eval_batch_size, eval_height, eval_width, 3))
      logits, _ = vgg.vgg_19(eval_inputs, is_training=False,
                             spatial_squeeze=False)
      self.assertListEqual(logits.get_shape().as_list(),
                           [eval_batch_size, 2, 2, num_classes])
      logits = tf.reduce_mean(logits, [1, 2])
      predictions = tf.argmax(logits, 1)
      self.assertEquals(predictions.get_shape().as_list(), [eval_batch_size]) 

def call(self, inputs, **kwargs):
    del kwargs
    features = inputs
    set_custom_getter_compose(self._custom_getter)
    tf.get_variable_scope().set_initializer(
        optimize.get_variable_initializer(self.hparams))
    with self._eager_var_store.as_default():
      self._fill_problem_hparams_features(features)
      sharded_features = self._shard_features(features)
      sharded_logits, losses = self.model_fn_sharded(sharded_features)
      if isinstance(sharded_logits, dict):
        concat_logits = {}
        for k, v in six.iteritems(sharded_logits):
          concat_logits[k] = tf.concat(v, 0)
        return concat_logits, losses
      else:
        return tf.concat(sharded_logits, 0), losses 

def testTrainEvalWithReuse(self):
    train_batch_size = 2
    eval_batch_size = 1
    train_height, train_width = 224, 224
    eval_height, eval_width = 256, 256
    num_classes = 1000
    with self.test_session():
      train_inputs = tf.random_uniform(
          (train_batch_size, train_height, train_width, 3))
      logits, _ = vgg.vgg_a(train_inputs)
      self.assertListEqual(logits.get_shape().as_list(),
                           [train_batch_size, num_classes])
      tf.get_variable_scope().reuse_variables()
      eval_inputs = tf.random_uniform(
          (eval_batch_size, eval_height, eval_width, 3))
      logits, _ = vgg.vgg_a(eval_inputs, is_training=False,
                            spatial_squeeze=False)
      self.assertListEqual(logits.get_shape().as_list(),
                           [eval_batch_size, 2, 2, num_classes])
      logits = tf.reduce_mean(logits, [1, 2])
      predictions = tf.argmax(logits, 1)
      self.assertEquals(predictions.get_shape().as_list(), [eval_batch_size]) 

def testTrainEvalWithReuse(self):
    train_batch_size = 2
    eval_batch_size = 1
    train_height, train_width = 224, 224
    eval_height, eval_width = 256, 256
    num_classes = 1000
    with self.test_session():
      train_inputs = tf.random_uniform(
          (train_batch_size, train_height, train_width, 3))
      logits, _ = vgg.vgg_a(train_inputs)
      self.assertListEqual(logits.get_shape().as_list(),
                           [train_batch_size, num_classes])
      tf.get_variable_scope().reuse_variables()
      eval_inputs = tf.random_uniform(
          (eval_batch_size, eval_height, eval_width, 3))
      logits, _ = vgg.vgg_a(eval_inputs, is_training=False,
                            spatial_squeeze=False)
      self.assertListEqual(logits.get_shape().as_list(),
                           [eval_batch_size, 2, 2, num_classes])
      logits = tf.reduce_mean(logits, [1, 2])
      predictions = tf.argmax(logits, 1)
      self.assertEquals(predictions.get_shape().as_list(), [eval_batch_size]) 

def testTrainEvalWithReuse(self):
    train_batch_size = 2
    eval_batch_size = 1
    train_height, train_width = 224, 224
    eval_height, eval_width = 256, 256
    num_classes = 1000
    with self.test_session():
      train_inputs = tf.random_uniform(
          (train_batch_size, train_height, train_width, 3))
      logits, _ = vgg.vgg_19(train_inputs)
      self.assertListEqual(logits.get_shape().as_list(),
                           [train_batch_size, num_classes])
      tf.get_variable_scope().reuse_variables()
      eval_inputs = tf.random_uniform(
          (eval_batch_size, eval_height, eval_width, 3))
      logits, _ = vgg.vgg_19(eval_inputs, is_training=False,
                             spatial_squeeze=False)
      self.assertListEqual(logits.get_shape().as_list(),
                           [eval_batch_size, 2, 2, num_classes])
      logits = tf.reduce_mean(logits, [1, 2])
      predictions = tf.argmax(logits, 1)
      self.assertEquals(predictions.get_shape().as_list(), [eval_batch_size]) 

def conv_tower_fn(self, images, is_training=True, reuse=None):
    """Computes convolutional features using the InceptionV3 model.

    Args:
      images: A tensor of shape [batch_size, height, width, channels].
      is_training: whether is training or not.
      reuse: whether or not the network and its variables should be reused. To
        be able to reuse 'scope' must be given.

    Returns:
      A tensor of shape [batch_size, OH, OW, N], where OWxOH is resolution of
      output feature map and N is number of output features (depends on the
      network architecture).
    """
    mparams = self._mparams['conv_tower_fn']
    logging.debug('Using final_endpoint=%s', mparams.final_endpoint)
    with tf.variable_scope('conv_tower_fn/INCE'):
      if reuse:
        tf.get_variable_scope().reuse_variables()
      with slim.arg_scope(inception.inception_v3_arg_scope()):
        net, _ = inception.inception_v3_base(
            images, final_endpoint=mparams.final_endpoint)
      return net 

def testTrainEvalWithReuse(self):
    train_batch_size = 5
    eval_batch_size = 2
    height, width = 150, 150
    num_classes = 1000
    with self.test_session() as sess:
      train_inputs = tf.random_uniform((train_batch_size, height, width, 3))
      inception.inception_v3(train_inputs, num_classes)
      tf.get_variable_scope().reuse_variables()
      eval_inputs = tf.random_uniform((eval_batch_size, height, width, 3))
      logits, _ = inception.inception_v3(eval_inputs, num_classes,
                                         is_training=False)
      predictions = tf.argmax(logits, 1)
      sess.run(tf.global_variables_initializer())
      output = sess.run(predictions)
      self.assertEquals(output.shape, (eval_batch_size,)) 

def testTrainEvalWithReuse(self):
    train_batch_size = 2
    eval_batch_size = 1
    train_height, train_width = 231, 231
    eval_height, eval_width = 281, 281
    num_classes = 1000
    with self.test_session():
      train_inputs = tf.random_uniform(
          (train_batch_size, train_height, train_width, 3))
      logits, _ = overfeat.overfeat(train_inputs)
      self.assertListEqual(logits.get_shape().as_list(),
                           [train_batch_size, num_classes])
      tf.get_variable_scope().reuse_variables()
      eval_inputs = tf.random_uniform(
          (eval_batch_size, eval_height, eval_width, 3))
      logits, _ = overfeat.overfeat(eval_inputs, is_training=False,
                                    spatial_squeeze=False)
      self.assertListEqual(logits.get_shape().as_list(),
                           [eval_batch_size, 2, 2, num_classes])
      logits = tf.reduce_mean(logits, [1, 2])
      predictions = tf.argmax(logits, 1)
      self.assertEquals(predictions.get_shape().as_list(), [eval_batch_size]) 

def __init__(self, name, window_size, obs_stack):
        self.window_size = window_size
        self.obs_stack = obs_stack

        with tf.variable_scope(name):
            self.input = tf.placeholder(dtype=tf.float32, shape=[None, window_size, window_size, obs_stack])
            self.conv1 = tf.layers.conv2d(inputs=self.input, filters=32, kernel_size=[8, 8], strides=[4, 4], padding='VALID', activation=tf.nn.relu)
            self.conv2 = tf.layers.conv2d(inputs=self.conv1, filters=64, kernel_size=[4, 4], strides=[2, 2], padding='VALID', activation=tf.nn.relu)
            self.conv3 = tf.layers.conv2d(inputs=self.conv2, filters=64, kernel_size=[3, 3], strides=[1, 1], padding='VALID', activation=tf.nn.relu)

            self.reshape = tf.reshape(self.conv3, [-1, 7 * 7 * 64])
            self.dense_3 = tf.layers.dense(inputs=self.reshape, units=512, activation=tf.nn.relu)
            
            self.critic = tf.layers.dense(inputs=self.dense_3, units=1, activation=None)

            self.scope = tf.get_variable_scope().name 

def top(self, body_output, _):
    # TODO(lukaszkaiser): is this a universal enough way to get channels?
    num_channels = self._model_hparams.problem.num_channels
    with tf.variable_scope("rgb_softmax"):
      body_output_shape = common_layers.shape_list(body_output)
      reshape_shape = body_output_shape[:3]
      reshape_shape.extend([num_channels, self.top_dimensionality])
      res = tf.layers.dense(body_output, self.top_dimensionality * num_channels)
      res = tf.reshape(res, reshape_shape)
      if not tf.get_variable_scope().reuse:
        res_argmax = tf.argmax(res, axis=-1)
        tf.summary.image(
            "result",
            common_layers.tpu_safe_image_summary(res_argmax),
            max_outputs=1)
      return res 

def top(self, body_output, _):
    num_channels = self._model_hparams.problem.num_channels
    num_frames = self._model_hparams.video_num_target_frames
    with tf.variable_scope("rgb_softmax"):
      body_output_shape = common_layers.shape_list(body_output)
      reshape_shape = body_output_shape[:3]
      reshape_shape.extend([num_channels, num_frames, self.top_dimensionality])
      res = tf.layers.dense(body_output,
                            self.top_dimensionality * num_channels * num_frames)
      res = tf.reshape(res, reshape_shape)
      res = tf.transpose(res, [0, 4, 1, 2, 3, 5])
      if not tf.get_variable_scope().reuse:
        res_argmax = tf.argmax(res[:, -1, :, :, :, :], axis=-1)
        tf.summary.image(
            "result",
            common_layers.tpu_safe_image_summary(res_argmax),
            max_outputs=1)
      return res 

def dense(x, size, name, weight_init=None, bias_init=0, weight_loss_dict=None, reuse=None):
    with tf.variable_scope(name, reuse=reuse):
        assert (len(tf.get_variable_scope().name.split('/')) == 2)

        w = tf.get_variable("w", [x.get_shape()[1], size], initializer=weight_init)
        b = tf.get_variable("b", [size], initializer=tf.constant_initializer(bias_init))
        weight_decay_fc = 3e-4

        if weight_loss_dict is not None:
            weight_decay = tf.multiply(tf.nn.l2_loss(w), weight_decay_fc, name='weight_decay_loss')
            if weight_loss_dict is not None:
                weight_loss_dict[w] = weight_decay_fc
                weight_loss_dict[b] = 0.0

            tf.add_to_collection(tf.get_variable_scope().name.split('/')[0] + '_' + 'losses', weight_decay)

        return tf.nn.bias_add(tf.matmul(x, w), b) 

def __init__(self, name, window_size, obs_stack, output_size, num_support):
        self.window_size = window_size
        self.obs_stack = obs_stack
        self.output_size = output_size
        self.num_support = num_support
        with tf.variable_scope(name):
            self.input = tf.placeholder(tf.float32, shape=[None, self.window_size, self.window_size, self.obs_stack])
            self.conv1 = tf.layers.conv2d(inputs=self.input, filters=32, kernel_size=[8, 8], strides=[4, 4], padding='VALID', activation=tf.nn.relu)
            self.conv2 = tf.layers.conv2d(inputs=self.conv1, filters=64, kernel_size=[4, 4], strides=[2, 2], padding='VALID', activation=tf.nn.relu)
            self.conv3 = tf.layers.conv2d(inputs=self.conv2, filters=64, kernel_size=[3, 3], strides=[1, 1], padding='VALID', activation=tf.nn.relu)
            self.reshape = tf.reshape(self.conv3, [-1, 7 * 7 * 64])
            self.l1 = tf.layers.dense(inputs=self.reshape, units=512, activation=tf.nn.relu)
            self.l2 = tf.layers.dense(inputs=self.l1, units=self.output_size * self.num_support, activation=None)
            self.net = tf.reshape(self.l2, [-1, self.output_size, self.num_support])

            self.scope = tf.get_variable_scope().name 

def assert_rank(tensor, expected_rank, name=None):
    """Raises an exception if the tensor rank is not of the expected rank.

    Args:
        tensor: A tf.Tensor to check the rank of.
        expected_rank: Python integer or list of integers, expected rank.
        name: Optional name of the tensor for the error message.

    Raises:
        ValueError: If the expected shape doesn't match the actual shape.
    """
    if name is None:
        name = tensor.name

    expected_rank_dict = {}
    if isinstance(expected_rank, six.integer_types):
        expected_rank_dict[expected_rank] = True
    else:
        for x in expected_rank:
            expected_rank_dict[x] = True

    actual_rank = tensor.shape.ndims
    if actual_rank not in expected_rank_dict:
        scope_name = tf.get_variable_scope().name
        raise ValueError(
                "For the tensor `%s` in scope `%s`, the actual rank "
                "`%d` (shape = %s) is not equal to the expected rank `%s`" %
                (name, scope_name, actual_rank, str(tensor.shape), str(expected_rank))) 

def __init__(self, name, window_size, obs_stack, output_size, lstm_units, lstm_layers):
        self.window_size = window_size
        self.output_size = output_size
        self.obs_stack = obs_stack
        self.reuse = []
        for i in range(self.obs_stack):
            if i == 0:
                self.reuse.append(False)
            else:
                self.reuse.append(True)

        self.lstm_list = [lstm_units for i in range(lstm_layers)]

        with tf.variable_scope(name):
            self.input = tf.placeholder(dtype=tf.float32, shape=[None, self.window_size, self.window_size, self.obs_stack])
            self.expand_input = tf.expand_dims(self.input, axis=3)
            self.split = [self.expand_input[:, :, :, :, i] for i in range(self.obs_stack)]
            self.conv1 = [tf.layers.conv2d(inputs=self.split[i], filters=8, kernel_size=[8, 8], strides=[4, 4], padding='VALID', activation=tf.nn.relu, name='conv1', reuse=self.reuse[i]) for i in range(self.obs_stack)]
            self.conv2 = [tf.layers.conv2d(inputs=self.conv1[i], filters=16, kernel_size=[4, 4], strides=[2, 2], padding='VALID', activation=tf.nn.relu, name='conv2', reuse=self.reuse[i]) for i in range(self.obs_stack)]
            self.conv3 = [tf.layers.conv2d(inputs=self.conv2[i], filters=16, kernel_size=[3, 3], strides=[1, 1], padding='VALID', activation=tf.nn.relu, name='conv3', reuse=self.reuse[i]) for i in range(self.obs_stack)]
            self.reshape = [tf.reshape(self.conv3[i], [-1, 7 * 7 * 16]) for i in range(self.obs_stack)]
            self.concat = tf.stack(self.reshape, axis=1)
            
            enc_cell = [tf.nn.rnn_cell.GRUCell(size) for size in self.lstm_list]
            enc_cell = tf.nn.rnn_cell.MultiRNNCell(enc_cell)
            self.outputs_enc, enc_states = tf.nn.dynamic_rnn(cell=enc_cell, inputs=self.concat, dtype=tf.float32)
            
            self.last_layer = self.outputs_enc[:, -1]
            self.actor = tf.layers.dense(inputs=self.last_layer, units=self.output_size, activation=tf.nn.softmax)

            self.scope = tf.get_variable_scope().name 

def __init__(self, name, state_size):
        self.state_size = state_size

        with tf.variable_scope(name):
            self.input = tf.placeholder(dtype=tf.float32, shape=[None, self.state_size])
            self.dense_1 = tf.layers.dense(inputs=self.input, units=256, activation=tf.nn.relu)
            self.dense_2 = tf.layers.dense(inputs=self.dense_1, units=256, activation=tf.nn.relu)
            self.dense_3 = tf.layers.dense(inputs=self.dense_2, units=256, activation=tf.nn.relu)
            self.critic = tf.layers.dense(inputs=self.dense_3, units=1, activation=None)

            self.scope = tf.get_variable_scope().name 

def _lstm_model(inputs, targets, config):
    '''
    @Use BasicLSTMCell and MultiRNNCell class to build LSTM model, 
    @return logits, cost and others
    '''
    batch_size = config.batch_size
    num_steps = config.num_steps
    num_layers = config.num_layers
    size = config.hidden_size
    vocab_size = config.vocab_size
    target_num = config.target_num # target output number    
    
    cell = tf.nn.rnn_cell.MultiRNNCell([tf.nn.rnn_cell.BasicLSTMCell(size) for _ in range(num_layers)])

    initial_state = cell.zero_state(batch_size, data_type())
    
    outputs = [] # outputs shape: list of tensor with shape [batch_size, size], length: time_step
    state = initial_state
    with tf.variable_scope("pos_lstm"):
        for time_step in range(num_steps):
            if time_step > 0: tf.get_variable_scope().reuse_variables()
            (cell_output, state) = cell(inputs[:, time_step, :], state) # inputs[batch_size, time_step, hidden_size]
            outputs.append(cell_output)
    
    output = tf.reshape(tf.concat(outputs, 1), [-1, size])    # output shape [time_step, size]
    softmax_w = tf.get_variable("softmax_w", [size, target_num], dtype=data_type())
    softmax_b = tf.get_variable("softmax_b", [target_num], dtype=data_type())
    logits = tf.matmul(output, softmax_w) + softmax_b         # logits shape[time_step, target_num]
    
    loss = tf.nn.sparse_softmax_cross_entropy_with_logits(labels = tf.reshape(targets, [-1]), logits = logits)
    cost = tf.reduce_sum(loss)/batch_size # loss [time_step]
    
    # adding extra statistics to monitor
    correct_prediction = tf.equal(tf.cast(tf.argmax(logits, 1), tf.int32), tf.reshape(targets, [-1]))
    accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float"))
    return cost, logits, state, initial_state 

def __init__(self, name, state_size, output_size):
        self.state_size = state_size
        self.output_size = output_size

        with tf.variable_scope(name):
            self.input = tf.placeholder(dtype=tf.float32, shape=[None, self.state_size])
            self.dense_1 = tf.layers.dense(inputs=self.input, units=256, activation=tf.nn.relu)
            self.dense_2 = tf.layers.dense(inputs=self.dense_1, units=256, activation=tf.nn.relu)
            self.dense_3 = tf.layers.dense(inputs=self.dense_2, units=256, activation=tf.nn.relu)
            self.actor = tf.layers.dense(inputs=self.dense_3, units=self.output_size, activation=tf.nn.softmax)

            self.scope = tf.get_variable_scope().name 

def __init__(self, name, state_size):
        self.state_size = state_size
        
        with tf.variable_scope(name):
            self.input = tf.placeholder(tf.float32, shape=[None, self.state_size])

            self.l1 = tf.layers.dense(inputs=self.input, units=128, activation=tf.nn.relu)
            self.l2 = tf.layers.dense(inputs=self.l1,    units=128, activation=tf.nn.relu)
            self.l3 = tf.layers.dense(inputs=self.l2,    units=128, activation=tf.nn.relu)
            
            self.value = tf.layers.dense(inputs=self.l3, units=1,   activation=None)
            self.v = tf.squeeze(self.value, axis=1)

            self.scope = tf.get_variable_scope().name 

def __init__(self, name, window_size, obs_stack, output_size):
        self.window_size = window_size
        self.obs_stack = obs_stack
        self.output_size = output_size
        with tf.variable_scope(name):
            self.input = tf.placeholder(tf.float32, shape=[None, self.window_size, self.window_size, self.obs_stack])
            self.conv1 = tf.layers.conv2d(inputs=self.input, filters=32, kernel_size=[8, 8], strides=[4, 4], padding='VALID', activation=tf.nn.relu)
            self.conv2 = tf.layers.conv2d(inputs=self.conv1, filters=64, kernel_size=[4, 4], strides=[2, 2], padding='VALID', activation=tf.nn.relu)
            self.conv3 = tf.layers.conv2d(inputs=self.conv2, filters=64, kernel_size=[3, 3], strides=[1, 1], padding='VALID', activation=tf.nn.relu)
            self.reshape = tf.reshape(self.conv3, [-1, 7 * 7 * 64])
            self.dense_3 = tf.layers.dense(inputs=self.reshape, units=512, activation=tf.nn.relu)
            self.Q = tf.layers.dense(inputs=self.dense_3, units=self.output_size, activation=None)

            self.scope = tf.get_variable_scope().name 

def __init__(self, name, window_size, obs_stack, output_size, num_support, batch_size):
        self.window_size = window_size
        self.obs_stack = obs_stack
        self.output_size = output_size
        self.num_support = num_support
        self.batch_size = batch_size
        self.quantile_embedding_dim = 128
        with tf.variable_scope(name):
            self.input = tf.placeholder(tf.float32, shape=[None, self.window_size, self.window_size, self.obs_stack])
            self.input_expand = tf.expand_dims(self.input, axis=1)
            self.input_tile = tf.tile(self.input_expand, [1, self.num_support, 1, 1, 1])
            self.input_reshape = tf.reshape(self.input_tile, [-1, self.window_size, self.window_size, self.obs_stack])
            self.conv1 = tf.layers.conv2d(inputs=self.input_reshape, filters=32, kernel_size=[8, 8], strides=[4, 4], padding='VALID', activation=tf.nn.relu)
            self.conv2 = tf.layers.conv2d(inputs=self.conv1, filters=64, kernel_size=[4, 4], strides=[2, 2], padding='VALID', activation=tf.nn.relu)
            self.conv3 = tf.layers.conv2d(inputs=self.conv2, filters=64, kernel_size=[3, 3], strides=[1, 1], padding='VALID', activation=tf.nn.relu)
            self.reshape = tf.reshape(self.conv3, [-1, 7 * 7 * 64])
            self.l1 = tf.layers.dense(inputs=self.reshape, units=self.quantile_embedding_dim, activation=tf.nn.relu)
            
            self.tau = tf.placeholder(tf.float32, [None, self.num_support])
            self.tau_reshape = tf.reshape(self.tau, [-1, 1])
            self.pi_mtx = tf.constant(np.expand_dims(np.pi * np.arange(0, 64), axis=0), dtype=tf.float32)
            self.cos_tau = tf.cos(tf.matmul(self.tau_reshape, self.pi_mtx))
            self.phi = tf.layers.dense(inputs=self.cos_tau, units=self.quantile_embedding_dim, activation=tf.nn.relu)
            self.net_sum = tf.multiply(self.l1, self.phi)
            self.net_l1 = tf.layers.dense(inputs=self.net_sum, units=512, activation=tf.nn.relu)
            self.net_l2 = tf.layers.dense(inputs=self.net_l1, units=256, activation=tf.nn.relu)
            self.net_l3 = tf.layers.dense(inputs=self.net_l2, units=self.output_size, activation=None)
            
            self.net_action = tf.transpose(tf.split(self.net_l3, 1, axis=0), perm=[0, 2, 1])

            self.net = tf.transpose(tf.split(self.net_l3, self.batch_size, axis=0), perm=[0, 2, 1])
            
            self.scope = tf.get_variable_scope().name 

def __init__(self, hparams=None):
        EncoderBase.__init__(self, hparams)

        with tf.variable_scope(self.variable_scope):
            if self._hparams.initializer:
                tf.get_variable_scope().set_initializer(
                    layers.get_initializer(self._hparams.initializer))

            self.multihead_attention_list = []
            self.poswise_networks = []
            for i in range(self._hparams.num_blocks):
                with tf.variable_scope("layer_{}".format(i)):

                    with tf.variable_scope('attention'):
                        mh_attn = MultiheadAttentionEncoder(
                            self._hparams.multihead_attention)
                        self.multihead_attention_list.append(mh_attn)

                        if self._hparams.dim != mh_attn.hparams.output_dim:
                            raise ValueError(
                                'The "dim" in the hparams of '
                                '"multihead_attention" should be equal to the '
                                '"dim" of TransformerEncoder')

                    pw_net = FeedForwardNetwork(
                        hparams=self._hparams['poswise_feedforward'])
                    final_dim = pw_net.hparams.layers[-1]['kwargs']['units']
                    if self._hparams.dim != final_dim:
                        raise ValueError(
                            'The output dimenstion of '
                            '"poswise_feedforward" should be equal '
                            'to the "dim" of TransformerEncoder.')
                    self.poswise_networks.append(pw_net) 

def __init__(self, name, state_size, output_size, num_support, batch_size):
        self.state_size = state_size
        self.output_size = output_size
        self.num_support = num_support
        self.quantile_embedding_dim = 128
        self.batch_size = batch_size
        with tf.variable_scope(name):
            self.input = tf.placeholder(tf.float32, shape=[None, self.state_size])
            self.tau = tf.placeholder(tf.float32, shape=[None, self.num_support])

            state_tile = tf.tile(self.input, [1, self.num_support])
            state_reshape = tf.reshape(state_tile, [-1, self.state_size])
            state_net = tf.layers.dense(inputs=state_reshape, units=self.quantile_embedding_dim, activation=tf.nn.selu)

            tau = tf.reshape(self.tau, [-1, 1])
            pi_mtx = tf.constant(np.expand_dims(np.pi * np.arange(0, 64), axis=0), dtype=tf.float32)
            cos_tau = tf.cos(tf.matmul(tau, pi_mtx))
            phi = tf.layers.dense(inputs=cos_tau, units=self.quantile_embedding_dim, activation=tf.nn.relu)

            net = tf.multiply(state_net, phi)
            net = tf.layers.dense(inputs=net, units=512, activation=tf.nn.relu)
            net = tf.layers.dense(inputs=net, units=128, activation=tf.nn.relu)
            net = tf.layers.dense(inputs=net, units=self.output_size, activation=None)

            self.net_action = tf.transpose(tf.split(net, 1, axis=0), perm=[0, 2, 1])

            self.net = tf.transpose(tf.split(net, self.batch_size, axis=0), perm=[0, 2, 1])

            self.scope = tf.get_variable_scope().name 

def __init__(self, name, state_size, output_size, num_support):
        self.state_size = state_size
        self.output_size = output_size
        self.num_support = num_support
        with tf.variable_scope(name):
            self.input = tf.placeholder(tf.float32, shape=[None, self.state_size])
            self.l1 = tf.layers.dense(inputs=self.input, units=256, activation=tf.nn.relu)
            self.l2 = tf.layers.dense(inputs=self.l1, units=256, activation=tf.nn.relu)
            self.l3 = tf.layers.dense(inputs=self.l2, units=self.output_size * self.num_support, activation=None)
            self.net = tf.reshape(self.l3, [-1, self.output_size, self.num_support])

            self.scope = tf.get_variable_scope().name 

def __init__(self, name, ob_space, ac_space, kind='large'):
        with tf.variable_scope(name):
            self._init(ob_space, ac_space, kind)
            self.scope = tf.get_variable_scope().name 

def assert_rank(tensor, expected_rank, name=None):
    """Raises an exception if the tensor rank is not of the expected rank.

    Args:
      tensor: A tf.Tensor to check the rank of.
      expected_rank: Python integer or list of integers, expected rank.
      name: Optional name of the tensor for the error message.

    Raises:
      ValueError: If the expected shape doesn't match the actual shape.
    """
    if name is None:
        name = tensor.name

    expected_rank_dict = {}
    if isinstance(expected_rank, six.integer_types):
        expected_rank_dict[expected_rank] = True
    else:
        for x in expected_rank:
            expected_rank_dict[x] = True

    actual_rank = tensor.shape.ndims
    if actual_rank not in expected_rank_dict:
        scope_name = tf.get_variable_scope().name
        raise ValueError(
            "For the tensor `%s` in scope `%s`, the actual rank "
            "`%d` (shape = %s) is not equal to the expected rank `%s`" %
            (name, scope_name, actual_rank, str(tensor.shape), str(expected_rank))) 

def __init__(self, name, window_size, obs_stack, output_size, lstm_units, lstm_layers):
        self.window_size = window_size
        self.output_size = output_size
        self.obs_stack = obs_stack
        self.reuse = []
        for i in range(self.obs_stack):
            if i == 0:
                self.reuse.append(False)
            else:
                self.reuse.append(True)

        self.lstm_list = [lstm_units for i in range(lstm_layers)]

        with tf.variable_scope(name):
            self.input = tf.placeholder(dtype=tf.float32, shape=[None, self.window_size, self.window_size, self.obs_stack])
            self.expand_input = tf.expand_dims(self.input, axis=3)
            self.split = [self.expand_input[:, :, :, :, i] for i in range(self.obs_stack)]
            self.conv1 = [tf.layers.conv2d(inputs=self.split[i], filters=8, kernel_size=[8, 8], strides=[4, 4], padding='VALID', activation=tf.nn.relu, name='conv1', reuse=self.reuse[i]) for i in range(self.obs_stack)]
            self.conv2 = [tf.layers.conv2d(inputs=self.conv1[i], filters=16, kernel_size=[4, 4], strides=[2, 2], padding='VALID', activation=tf.nn.relu, name='conv2', reuse=self.reuse[i]) for i in range(self.obs_stack)]
            self.conv3 = [tf.layers.conv2d(inputs=self.conv2[i], filters=16, kernel_size=[3, 3], strides=[1, 1], padding='VALID', activation=tf.nn.relu, name='conv3', reuse=self.reuse[i]) for i in range(self.obs_stack)]
            self.reshape = [tf.reshape(self.conv3[i], [-1, 7 * 7 * 16]) for i in range(self.obs_stack)]
            self.concat = tf.stack(self.reshape, axis=1)
            
            enc_cell = [tf.nn.rnn_cell.GRUCell(size) for size in self.lstm_list]
            enc_cell = tf.nn.rnn_cell.MultiRNNCell(enc_cell)
            self.outputs_enc, enc_states = tf.nn.dynamic_rnn(cell=enc_cell, inputs=self.concat, dtype=tf.float32)
            
            self.last_layer = self.outputs_enc[:, -1]
            self.critic = tf.layers.dense(inputs=self.last_layer, units=1, activation=None)


            self.scope = tf.get_variable_scope().name