Python tensorflow.compat.v1.gradients() Examples

def minimize(self, loss, x, optim_state):
    """Compute a new value of `x` to minimize `loss`.

    Args:
      loss: A scalar Tensor, the value to be minimized. `loss` should be a
        continuous function of `x` which supports gradients, `loss = f(x)`.
      x: A list of Tensors, the values to be updated. This is analogous to the
        `var_list` argument in standard TF Optimizer.
      optim_state: A (possibly nested) dict, containing any state info needed
        for the optimizer.

    Returns:
      new_x: A list of Tensors, the same length as `x`, which are updated
      new_optim_state: A new state, with the same structure as `optim_state`,
        which have been updated.
    """ 

def testGradients(self):
    with self.test_session() as sess:
      policy_logits = tf.constant([[0, 1], [0, 1], [1, 1], [0, 100]],
                                  dtype=tf.float32)
      action_values = tf.constant([0, 1, 2, 1], dtype=tf.float32)
      actions = tf.constant([0, 0, 1, 1], dtype=tf.int32)
      loss = pg_ops.discrete_policy_gradient(policy_logits, actions,
                                             action_values)
      total_loss = tf.reduce_sum(loss)
      gradients = tf.gradients([total_loss], [policy_logits])
      grad_policy_logits = sess.run(gradients[0])
      #  The final case (with large logits), runs out of precision and gets
      #  truncated to 0, but isn't `nan`.
      self.assertAllClose(grad_policy_logits,
                          [[0, 0], [-0.731, 0.731], [1, -1], [0, 0]], atol=1e-4)

      self.assertAllEqual(tf.gradients([total_loss], [actions, action_values]),
                          [None, None]) 

def testGradients(self, is_multi_actions):
    self._setUpLoss(is_multi_actions)
    with self.test_session() as sess:
      total_loss = tf.reduce_sum(self._loss)
      gradients = tf.gradients(
          [total_loss], nest.flatten(self._policy_logits_nest))
      grad_policy_logits_nest = sess.run(gradients)
      for grad_policy_logits in grad_policy_logits_nest:
        self.assertAllClose(grad_policy_logits,
                            [[[0, 0], [-0.731, 0.731]],
                             [[1, -1], [0, 0]]], atol=1e-4)
      dead_grads = tf.gradients(
          [total_loss],
          nest.flatten(self._actions_nest) + [self._action_values])
      for grad in dead_grads:
        self.assertIsNone(grad) 

def testEntropyGradients(self, is_multi_actions):
    if is_multi_actions:
      loss = self.multi_op.extra.entropy_loss
      policy_logits_nest = self.multi_policy_logits
    else:
      loss = self.op.extra.entropy_loss
      policy_logits_nest = self.policy_logits

    grad_policy_list = [
        tf.gradients(loss, policy_logits)[0] * self.num_actions
        for policy_logits in nest.flatten(policy_logits_nest)]

    for grad_policy in grad_policy_list:
      self.assertEqual(grad_policy.get_shape(), tf.TensorShape([2, 1, 3]))

    self.assertAllEqual(tf.gradients(loss, self.baseline_values), [None])
    self.assertAllEqual(tf.gradients(loss, self.invalid_grad_inputs),
                        self.invalid_grad_outputs) 

def test_loss_gradient_conv2(self):
    loss = self.gamma_activation_reg.get_regularization_term(
        [self.get_conv('conv2')])
    expected_grad = np.array([-1.0] * 11 + [1.0] * 12)
    gammas = [
        self.name_to_var['conv%d/BatchNorm/gamma' % i] for i in range(1, 5)
    ]

    # Although the loss associated with conv2 depends on the gammas of conv2,
    # conv3 and conv4, only gamma2 should receive graients. The other gammas are
    # compared to the threshold to see if they are alive or not, but this should
    # not send gradients to them.
    grads = tf.gradients(loss, gammas)
    self.assertEqual(None, grads[0])
    self.assertEqual(None, grads[2])
    self.assertEqual(None, grads[3])

    # Regarding gamma2, it receives a -1 or a +1 gradient, depending on whether
    # the gamma is negative or positive, since the loss is |gamma|. This is
    # multiplied by expected_coeff.
    with self.cached_session():
      self.assertAllClose(expected_grad, grads[1].eval()) 

def testGradient(self):
    with self.test_session() as sess:
      x = tf.placeholder(tf.float64)
      quadratic_linear_boundary = 3
      loss = clipping_ops.huber_loss(x, quadratic_linear_boundary)
      xs = np.array([-5, -4, -3.1, -3, -2.9, 2, -1, 0, 1, 2, 2.9, 3, 3.1, 4, 5])
      grads = sess.run(tf.gradients([loss], [x]), feed_dict={x: xs})[0]

    self.assertTrue(np.all(np.abs(grads) <= quadratic_linear_boundary))

    # Everything <= -3 should have gradient -3.
    grads_lo = grads[xs <= -quadratic_linear_boundary]
    self.assertAllEqual(grads_lo,
                        [-quadratic_linear_boundary] * grads_lo.shape[0])

    # Everything >= 3 should have gradient 3.
    grads_hi = grads[xs >= quadratic_linear_boundary]
    self.assertAllEqual(grads_hi,
                        [quadratic_linear_boundary] * grads_hi.shape[0])

    # x in (-3, 3) should have gradient x.
    grads_mid = grads[np.abs(xs) <= quadratic_linear_boundary]
    xs_mid = xs[np.abs(xs) <= quadratic_linear_boundary]
    self.assertAllEqual(grads_mid, xs_mid) 

def recompute_grad(fn):
  """Decorator that recomputes the function on the backwards pass.

  Args:
    fn: a function that takes Tensors (all as positional arguments) and returns
      a tuple of Tensors.

  Returns:
    A wrapped fn that is identical to fn when called, but its activations will
    be discarded and recomputed on the backwards pass (i.e. on a call to
    tf.gradients).
  """

  @functools.wraps(fn)
  def wrapped(*args):
    return _recompute_grad(fn, args)

  return wrapped 

def test_adam(self):
    with self.test_session() as sess:
      w = tf.get_variable(
          "w",
          shape=[3],
          initializer=tf.constant_initializer([0.1, -0.2, -0.1]))
      x = tf.constant([0.4, 0.2, -0.5])
      loss = tf.reduce_mean(tf.square(x - w))
      tvars = tf.trainable_variables()
      grads = tf.gradients(loss, tvars)
      global_step = tf.train.get_or_create_global_step()
      optimizer = optimization.AdamWeightDecayOptimizer(learning_rate=0.2)
      train_op = optimizer.apply_gradients(list(zip(grads, tvars)), global_step)
      init_op = tf.group(tf.global_variables_initializer(),
                         tf.local_variables_initializer())
      sess.run(init_op)
      for _ in range(100):
        sess.run(train_op)
      w_np = sess.run(w)
      self.assertAllClose(w_np.flat, [0.4, 0.2, -0.5], rtol=1e-2, atol=1e-2) 

def _apply_gradients(self, grads, x, optim_state):
    """Applies gradients."""
    lr = self._lr_fn(optim_state.t)
    new_optim_state = self._State(
        t=optim_state.t + 1,
        m=[None] * len(x),
        u=[None] * len(x))
    t = tf.cast(new_optim_state.t, tf.float32)
    new_x = [None] * len(x)
    for i in range(len(x)):
      g = grads[i]
      m_old = optim_state.m[i]
      u_old = optim_state.u[i]
      new_optim_state.m[i] = self._beta1 * m_old + (1. - self._beta1) * g
      new_optim_state.u[i] = self._beta2 * u_old + (1. - self._beta2) * g * g
      m_hat = new_optim_state.m[i] / (1. - tf.pow(self._beta1, t))
      u_hat = new_optim_state.u[i] / (1. - tf.pow(self._beta2, t))
      new_x[i] = x[i] - lr * m_hat / (tf.sqrt(u_hat) + self._epsilon)
    return new_x, new_optim_state 

def _test_upper_bound(self, gradient):
    inputs = tf.placeholder(dtype=tf.float32)
    outputs = math_ops.upper_bound(inputs, 0, gradient=gradient)
    pgrads, = tf.gradients([outputs], [inputs], [tf.ones_like(inputs)])
    ngrads, = tf.gradients([outputs], [inputs], [-tf.ones_like(inputs)])

    inputs_feed = [-1, 1]
    outputs_expected = [-1, 0]
    if gradient == "disconnected":
      pgrads_expected = [1, 0]
      ngrads_expected = [-1, 0]
    elif gradient == "identity":
      pgrads_expected = [1, 1]
      ngrads_expected = [-1, -1]
    else:
      pgrads_expected = [1, 1]
      ngrads_expected = [-1, 0]

    with self.cached_session() as sess:
      outputs, pgrads, ngrads = sess.run(
          [outputs, pgrads, ngrads], {inputs: inputs_feed})
      self.assertAllEqual(outputs, outputs_expected)
      self.assertAllEqual(pgrads, pgrads_expected)
      self.assertAllEqual(ngrads, ngrads_expected) 

def _test_lower_bound(self, gradient):
    inputs = tf.placeholder(dtype=tf.float32)
    outputs = math_ops.lower_bound(inputs, 0, gradient=gradient)
    pgrads, = tf.gradients([outputs], [inputs], [tf.ones_like(inputs)])
    ngrads, = tf.gradients([outputs], [inputs], [-tf.ones_like(inputs)])

    inputs_feed = [-1, 1]
    outputs_expected = [0, 1]
    if gradient == "disconnected":
      pgrads_expected = [0, 1]
      ngrads_expected = [0, -1]
    elif gradient == "identity":
      pgrads_expected = [1, 1]
      ngrads_expected = [-1, -1]
    else:
      pgrads_expected = [0, 1]
      ngrads_expected = [-1, -1]

    with self.cached_session() as sess:
      outputs, pgrads, ngrads = sess.run(
          [outputs, pgrads, ngrads], {inputs: inputs_feed})
      self.assertAllEqual(outputs, outputs_expected)
      self.assertAllEqual(pgrads, pgrads_expected)
      self.assertAllEqual(ngrads, ngrads_expected) 

def testGradientsPolicyGradientLoss(self, multi_actions):
    self._setUp_a2c_loss(multi_actions=multi_actions)
    loss = self._extra.policy_gradient_loss

    for policy_var in nest.flatten(self._policy_vars):
      gradient = tf.gradients(loss, policy_var)[0]
      self.assertEqual(gradient.get_shape(), policy_var.get_shape())

    self.assertAllEqual(tf.gradients(loss, self._baseline_values), [None]) 

def testNoOtherGradients(self):
    # Gradients are only defined for logits_q_tm1, not any other input.
    # Bellman residual variants could potentially generate a gradient wrt q_t.
    gradients = tf.gradients([self.qlearning.loss],
                             [self.logits_q_t, self.r_t, self.a_tm1,
                              self.pcont_t, self.atoms_t, self.atoms_tm1,
                              self.q_t_selector])
    self.assertEqual(gradients, [None for _ in gradients]) 

def testDpgNoOtherGradients(self):
    """No gradient of loss w.r.t. parameters other than that of actor network.
    """
    with self.test_session():
      gradients = tf.gradients([self.loss], [self.q_tm1_max, self.w, self.b])
      self.assertListEqual(gradients, [None] * len(gradients)) 

def testInvalidGradients(self, multi_actions, gae_lambda):
    self._setUp_a2c_loss(multi_actions=multi_actions, gae_lambda=gae_lambda)
    ins = nest.flatten(
        [self._actions, self._rewards, self._pcontinues, self._bootstrap_value])
    outs = [None] * len(ins)

    self.assertAllEqual(tf.gradients(
        self._extra.discounted_returns, ins), outs)
    self.assertAllEqual(tf.gradients(
        self._extra.policy_gradient_loss, ins), outs)
    self.assertAllEqual(tf.gradients(self._extra.entropy_loss, ins), outs)
    self.assertAllEqual(tf.gradients(self._extra.baseline_loss, ins), outs)
    self.assertAllEqual(tf.gradients(self._loss, ins), outs) 

def testGradQtm1(self):
    with self.test_session() as sess:
      # Take gradients of the negative loss, so that the tests here check the
      # values propagated during gradient _descent_, rather than _ascent_.
      gradients = tf.gradients([-self.qlearning.loss], [self.logits_q_tm1])
      grad_q_tm1 = sess.run(gradients[0])
      # Correct gradient directions (including 0.0 for unused actions at t=tm1).
      expected = np.zeros_like(grad_q_tm1)
      expected[0, 2] = [-1, -1, 1]
      expected[1, 1] = [-1, 1, -1]
      expected[2, 3] = [-1, -1, 1]
      expected[3, 0] = [-1, 1, -1]
      expected[4, 1] = [-1, 1, -1]
      self.assertAllClose(np.sign(grad_q_tm1), expected) 

def testGradQtm1(self):
    with self.test_session() as sess:
      # Take gradients of the negative loss, so that the tests here check the
      # values propagated during gradient _descent_, rather than _ascent_.
      gradients = tf.gradients([-self.qlearning.loss], [self.logits_q_tm1])
      grad_q_tm1 = sess.run(gradients[0])
      # Correct gradient directions (including 0.0 for unused actions at t=tm1).
      expected = np.zeros_like(grad_q_tm1)
      expected[0, 2] = [-1, -1, 1]
      expected[1, 1] = [-1, 1, -1]
      expected[2, 3] = [-1, -1, 1]
      expected[3, 0] = [-1, 1, -1]
      expected[4, 1] = [-1, 1, -1]
      self.assertAllClose(np.sign(grad_q_tm1), expected) 

def testPersistentQLearningGradQtm1(self):
    with self.test_session() as sess:
      # Take gradients of the negative loss, so that the tests here check the
      # values propogated during gradient _descent_, rather than _ascent_.
      gradients = tf.gradients([-self.persistent_qlearning.loss], [self.q_tm1])
      grad_q_tm1 = sess.run(gradients[0])
      self.assertAllClose(grad_q_tm1, [[2, 0], [0, 17], [0, -1]]) 

def testGradient(self, multi_actions, normalise):
    sequence_length = 4
    batch_size = 2
    self._setUp_entropy(
        multi_actions, normalise, sequence_length, batch_size)
    loss = self._policy_entropy_loss.loss
    # MVN mu has None gradient
    self.assertIsNone(tf.gradients(loss, nest.flatten(self._policy_vars)[0])[0])
    for policy_var in nest.flatten(self._policy_vars)[1:]:
      gradient = tf.gradients(loss, policy_var)[0]
      self.assertEqual(gradient.get_shape(), policy_var.get_shape()) 

def testGradients(self, sequence_length, batch_size, action_dim):
    self._setUp_loss(sequence_length, batch_size, action_dim)
    total_loss = tf.reduce_sum(self._loss)
    for policy_var in self._policy_vars:
      gradients = tf.gradients(total_loss, policy_var)
      self.assertEqual(gradients[0].get_shape().as_list(),
                       policy_var.get_shape().as_list())
    gradients = tf.gradients([total_loss], [self._actions, self._action_values])
    self.assertEqual(gradients, [None, None]) 

def testNoOtherGradients(self):
    """Tests no gradient propagates through any tensors other than `q_tm1`."""
    # Gradients are only defined for q_tm1, not any other input.
    # Bellman residual variants could potentially generate a gradient wrt q_t.
    gradients = tf.gradients([self.loss_op],
                             [self.r_t, self.a_tm1, self.pcont_t, self.v_t])
    self.assertEqual(gradients, [None, None, None, None]) 

def testGradQtm1(self):
    """Tests that the gradients of negative loss are equal to the td_error."""
    with self.test_session() as sess:
      # Take gradients of the negative loss, so that the tests here check the
      # values propogated during gradient _descent_, rather than _ascent_.
      gradients = tf.gradients([-self.loss_op], [self.q_tm1])
      grad_q_tm1 = sess.run(gradients[0])
      self.assertAllClose(grad_q_tm1, [[0, 0, 0], [0, 3, 0]]) 

def testNoOtherGradients(self):
    # Gradients are only defined for q_tm1, not any other input.
    # Bellman residual variants could potentially generate a gradient wrt q_t.
    gradients = tf.gradients(
        [self.sarsa.loss],
        [self.q_t, self.r_t, self.a_tm1, self.pcont_t, self.a_t, self.lambda_])
    self.assertEqual(gradients, [None, None, None, None, None, None]) 

def testGradQtm1(self):
    with self.test_session() as sess:
      gradients = tf.gradients([-self.qlearning.loss], [self.q_tm1])
      gradients_reference = tf.gradients([-self.qlearning_reference.loss],
                                         [self.q_tm1])
      self.assertAllClose(
          sess.run(gradients[0]), sess.run(gradients_reference[0])) 

def testGradQtm1(self):
    with self.test_session() as sess:
      # Take gradients of the negative loss, so that the tests here check the
      # values propagated during gradient _descent_, rather than _ascent_.
      gradients = tf.gradients([-self.qlearning.loss], [self.q_tm1])
      grad_q_tm1 = sess.run(gradients[0])
      self.assertAllClose(grad_q_tm1,
                          [[[self.t00 - 1.1, 0], [0, self.t01 - 3.1]],
                           [[0, self.t10 - 1.1], [self.t11 + 1.1, 0]],
                           [[self.t20 - 3.1, 0], [self.t21 + 2.1, 0]]]) 

def testNoOtherGradients(self):
    """Tests no gradient propagates through any tensors other than q_tm1."""
    # Gradients are only defined for q_tm1, not any other input.
    # Bellman residual variants could potentially generate a gradient wrt q_t.
    gradients = tf.gradients(
        [self.sarse.loss],
        [self.q_t, self.r_t, self.a_tm1, self.pcont_t, self.probs_a_t])
    self.assertEqual(gradients, [None, None, None, None, None]) 

def testGradQtm1(self):
    """Tests that the gradients of negative loss are equal to the td_error."""
    with self.test_session() as sess:
      # Take gradients of the negative loss, so that the tests here check the
      # values propogated during gradient _descent_, rather than _ascent_.
      gradients = tf.gradients([-self.sarse.loss], [self.q_tm1])
      grad_q_tm1 = sess.run(gradients[0])
      self.assertAllClose(grad_q_tm1, [[4.4, 0, 0], [0, 2, 0]]) 

def testNoOtherGradients(self):
    """Tests no gradient propagates through any tensors other than q_tm1."""
    # Gradients are only defined for q_tm1, not any other input.
    # Bellman residual variants could potentially generate a gradient wrt q_t.
    gradients = tf.gradients(
        [self.sarsa.loss],
        [self.q_t, self.r_t, self.a_tm1, self.pcont_t, self.a_t])
    self.assertEqual(gradients, [None, None, None, None, None]) 

def testGradQtm1(self):
    """Tests that the gradients of negative loss are equal to the td_error."""
    with self.test_session() as sess:
      # Take gradients of the negative loss, so that the tests here check the
      # values propogated during gradient _descent_, rather than _ascent_.
      gradients = tf.gradients([-self.sarsa.loss], [self.q_tm1])
      grad_q_tm1 = sess.run(gradients[0])
      self.assertAllClose(grad_q_tm1, [[0, 0, 0], [0, 3, 0]]) 

def testGradQtm1(self):
    with self.test_session() as sess:
      # Take gradients of the negative loss, so that the tests here check the
      # values propogated during gradient _descent_, rather than _ascent_.
      gradients = tf.gradients([-self.qlearning.loss], [self.q_tm1])
      grad_q_tm1 = sess.run(gradients[0])
      self.assertAllClose(grad_q_tm1, [[0, 0, 0], [0, 1, 0]])