Python tensorflow.extract_image_patches() Examples

def create_conv_hessian_computing_tf_graph(input_shape, layer_kernel, layer_stride):
    """ 
    This function create the TensorFlow graph for computing hessian matrix for fully-connected layer.
    Compared with create_res_hessian_computing_tf_graph, it append extract one for bias term.
    """ 
    input_holder = tf.placeholder(dtype=tf.float32, shape=input_shape)
    patches = tf.extract_image_patches(images = input_holder,
                                       ksizes = [1,layer_kernel, layer_kernel,1],
                                       strides = [1, layer_stride, layer_stride, 1],
                                       rates = [1, 1, 1, 1],
                                       padding = 'SAME')
    print 'Patches shape: %s' %patches.get_shape()
    vect_w_b = tf.concat([patches, tf.ones([tf.shape(patches)[0], \
                tf.shape(patches)[1], tf.shape(patches)[2], 1])], axis=3)
    a = tf.expand_dims(vect_w_b, axis=-1)
    b = tf.expand_dims(vect_w_b, axis=3)
    outprod = tf.multiply(a, b)
    # print 'outprod shape: %s' %outprod.get_shape()
    get_hessian_op = tf.reduce_mean(outprod, axis=[0, 1, 2])
    print 'Hessian shape: %s' % get_hessian_op.get_shape()
    return input_holder, get_hessian_op


# Construct hessian inverse computing graph for Woodbury 

def patch_decomposition(self, T_features):
        # patch decomposition
        patch_size = 1
        patches_as_depth_vectors = tf.extract_image_patches(
            images=T_features, ksizes=[1, patch_size, patch_size, 1],
            strides=[1, 1, 1, 1], rates=[1, 1, 1, 1], padding='VALID',
            name='patches_as_depth_vectors')

        self.patches_NHWC = tf.reshape(
            patches_as_depth_vectors,
            shape=[-1, patch_size, patch_size, patches_as_depth_vectors.shape[3].value],
            name='patches_PHWC')

        self.patches_HWCN = tf.transpose(
            self.patches_NHWC,
            perm=[1, 2, 3, 0],
            name='patches_HWCP')  # tf.conv2 ready format

        return self.patches_HWCN 

def patch_decomposition(self, T_features):
        # patch decomposition
        patch_size = 1
        patches_as_depth_vectors = tf.extract_image_patches(
            images=T_features, ksizes=[1, patch_size, patch_size, 1],
            strides=[1, 1, 1, 1], rates=[1, 1, 1, 1], padding='VALID',
            name='patches_as_depth_vectors')

        self.patches_NHWC = tf.reshape(
            patches_as_depth_vectors,
            shape=[-1, patch_size, patch_size, patches_as_depth_vectors.shape[3].value],
            name='patches_PHWC')

        self.patches_HWCN = tf.transpose(
            self.patches_NHWC,
            perm=[1, 2, 3, 0],
            name='patches_HWCP')  # tf.conv2 ready format

        return self.patches_HWCN 

def kernel_tile(input, kernel, stride):
    # output = tf.extract_image_patches(input, ksizes=[1, kernel, kernel, 1], strides=[1, stride, stride, 1], rates=[1, 1, 1, 1], padding='VALID')

    input_shape = input.get_shape()
    tile_filter = np.zeros(shape=[kernel, kernel, input_shape[3],
                                  kernel * kernel], dtype=np.float32)
    for i in range(kernel):
        for j in range(kernel):
            tile_filter[i, j, :, i * kernel + j] = 1.0

    tile_filter_op = tf.constant(tile_filter, dtype=tf.float32)
    output = tf.nn.depthwise_conv2d(input, tile_filter_op, strides=[
                                    1, stride, stride, 1], padding='VALID')
    output_shape = output.get_shape()
    output = tf.reshape(output, shape=[int(output_shape[0]), int(
        output_shape[1]), int(output_shape[2]), int(input_shape[3]), kernel * kernel])
    output = tf.transpose(output, perm=[0, 1, 2, 4, 3])

    return output

# input should be a tensor with size as [batch_size, caps_num_i, 16] 

def compute_cost_volume(x1, x2, H, W, channel, d=9):
    x1 = tf.nn.l2_normalize(x1, axis=3)
    x2 = tf.nn.l2_normalize(x2, axis=3)        

    # choice 1: use tf.extract_image_patches, may not work for some tensorflow versions
    x2_patches = tf.extract_image_patches(x2, [1, d, d, 1], strides=[1, 1, 1, 1], rates=[1, 1, 1, 1], padding='SAME')
    
    # choice 2: use convolution, but is slower than choice 1
    # out_channels = d * d
    # w = tf.eye(out_channels*channel, dtype=tf.float32)
    # w = tf.reshape(w, (d, d, channel, out_channels*channel))
    # x2_patches = tf.nn.conv2d(x2, w, strides=[1, 1, 1, 1], padding='SAME')
    
    x2_patches = tf.reshape(x2_patches, [-1, H, W, d, d, channel])
    x1_reshape = tf.reshape(x1, [-1, H, W, 1, 1, channel])
    x1_dot_x2 = tf.multiply(x1_reshape, x2_patches)
    
    cost_volume = tf.reduce_sum(x1_dot_x2, axis=-1)
    #cost_volume = tf.reduce_mean(x1_dot_x2, axis=-1)
    cost_volume = tf.reshape(cost_volume, [-1, H, W, d*d])   
    return cost_volume 

def extract_patches_fn(image: tf.Tensor, patch_shape: Tuple[int, int], offsets: Tuple[int, int]) -> tf.Tensor:
    """Will cut a given image into patches.

    :param image: tf.Tensor
    :param patch_shape: shape of the extracted patches [h, w]
    :param offsets: offset to add to the origin of first patch top-right coordinate, useful during data augmentation \
    to have slighlty different patches each time. This value will be multiplied by [h/2, w/2] (range values [0,1])
    :return: patches [batch_patches, h, w, c]
    """
    with tf.name_scope('patch_extraction'):
        h, w = patch_shape
        c = image.get_shape()[-1]

        offset_h = tf.cast(tf.round(offsets[0] * h // 2), dtype=tf.int32)
        offset_w = tf.cast(tf.round(offsets[1] * w // 2), dtype=tf.int32)
        offset_img = image[offset_h:, offset_w:, :]
        offset_img = offset_img[None, :, :, :]

        patches = tf.extract_image_patches(offset_img, ksizes=[1, h, w, 1], strides=[1, h // 2, w // 2, 1],
                                           rates=[1, 1, 1, 1], padding='VALID')
        patches_shape = tf.shape(patches)
        return tf.reshape(patches, [tf.reduce_prod(patches_shape[:3]), h, w, int(c)]) 

def _VerifyValues(self, image, ksizes, strides, rates, padding, patches):
    """Tests input-output pairs for the ExtractImagePatches op.

    Args:
      image: Input tensor with shape: [batch, in_rows, in_cols, depth].
      ksizes: Patch size specified as: [ksize_rows, ksize_cols].
      strides: Output strides, specified as [stride_rows, stride_cols].
      rates: Atrous rates, specified as [rate_rows, rate_cols].
      padding: Padding type.
      patches: Expected output.
    """
    ksizes = [1] + ksizes + [1]
    strides = [1] + strides + [1]
    rates = [1] + rates + [1]

    with self.test_session(use_gpu=True):
      out_tensor = tf.extract_image_patches(
          tf.constant(image),
          ksizes=ksizes,
          strides=strides,
          rates=rates,
          padding=padding,
          name="im2col")
      self.assertAllClose(patches, out_tensor.eval()) 

def extract_image_patches(x, ksizes, ssizes, padding='same',
                          data_format='channels_last'):
    """Extract the patches from an image.

    # Arguments
        x: The input image
        ksizes: 2-d tuple with the kernel size
        ssizes: 2-d tuple with the strides size
        padding: 'same' or 'valid'
        data_format: 'channels_last' or 'channels_first'

    # Returns
        The (k_w,k_h) patches extracted
        TF ==> (batch_size,w,h,k_w,k_h,c)
        TH ==> (batch_size,w,h,c,k_w,k_h)
    """
    kernel = [1, ksizes[0], ksizes[1], 1]
    strides = [1, ssizes[0], ssizes[1], 1]
    padding = _preprocess_padding(padding)
    if data_format == 'channels_first':
        x = K.permute_dimensions(x, (0, 2, 3, 1))
    bs_i, w_i, h_i, ch_i = K.int_shape(x)
    patches = tf.extract_image_patches(x, kernel, strides, [1, 1, 1, 1],
                                       padding)
    # Reshaping to fit Theano
    bs, w, h, ch = K.int_shape(patches)
    reshaped = tf.reshape(patches, [-1, w, h, tf.floordiv(ch, ch_i), ch_i])
    final_shape = [-1, w, h, ch_i, ksizes[0], ksizes[1]]
    patches = tf.reshape(tf.transpose(reshaped, [0, 1, 2, 4, 3]), final_shape)
    if data_format == 'channels_last':
        patches = K.permute_dimensions(patches, [0, 1, 2, 4, 5, 3])
    return patches 

def forward(self):
        inp = self.inp.out
        s = self.lay.stride
        self.out = tf.extract_image_patches(
            inp, [1,s,s,1], [1,s,s,1], [1,1,1,1], 'VALID') 

def apply_dna_kernels(image, kernels, dilation_rate=(1, 1)):
    """
    Args:
        image: A 4-D tensor of shape
            `[batch, in_height, in_width, in_channels]`.
        kernels: A 6-D of shape
            `[batch, in_height, in_width, kernel_size[0], kernel_size[1], num_transformed_images]`.

    Returns:
        A list of `num_transformed_images` 4-D tensors, each of shape
            `[batch, in_height, in_width, in_channels]`.
    """
    dilation_rate = list(dilation_rate) if isinstance(dilation_rate, (tuple, list)) else [dilation_rate] * 2
    batch_size, height, width, color_channels = image.get_shape().as_list()
    batch_size, height, width, kernel_height, kernel_width, num_transformed_images = kernels.get_shape().as_list()
    kernel_size = [kernel_height, kernel_width]

    # Flatten the spatial dimensions.
    kernels_reshaped = tf.reshape(kernels, [batch_size, height, width,
                                            kernel_size[0] * kernel_size[1], num_transformed_images])
    image_padded = pad2d(image, kernel_size, rate=dilation_rate, padding='SAME', mode='SYMMETRIC')
    # Combine channel and batch dimensions into the first dimension.
    image_transposed = tf.transpose(image_padded, [3, 0, 1, 2])
    image_reshaped = flatten(image_transposed, 0, 1)[..., None]
    patches_reshaped = tf.extract_image_patches(image_reshaped, ksizes=[1] + kernel_size + [1],
                                                strides=[1] * 4, rates=[1] + dilation_rate + [1], padding='VALID')
    # Separate channel and batch dimensions, and move channel dimension.
    patches_transposed = tf.reshape(patches_reshaped, [color_channels, batch_size, height, width, kernel_size[0] * kernel_size[1]])
    patches = tf.transpose(patches_transposed, [1, 2, 3, 0, 4])
    # Reduce along the spatial dimensions of the kernel.
    outputs = tf.matmul(patches, kernels_reshaped)
    outputs = tf.unstack(outputs, axis=-1)
    return outputs 

def extract_random_patches(image, patch_sizes, num_patches):
  """Extracts random patches from an image tensor."""
  if isinstance(patch_sizes, int):
    ksizes = [1, patch_sizes, patch_sizes, 1]
  elif isinstance(patch_sizes, list) or isinstance(patch_sizes, tuple):
    if len(patch_sizes) == 2:
      ksizes = [1, patch_sizes[0], patch_sizes[1], 1]
    elif len(patch_sizes) == 4:
      ksizes = patch_sizes
    else:
      raise ValueError('patch_sizes must be length 2 or length 4.')
  else:
    raise ValueError('patch_sizes must be a length 2 or length 4 list, or an int.')

  # (batch, height, width, patch_size * patch_size * feature)
  patches = tf.extract_image_patches(image, ksizes=ksizes,
                                     strides=[1, 1, 1, 1], rates=[1, 1, 1, 1], padding='VALID')
  patches_shape = tf.unstack(tf.shape(patches))
  patches_hw = patches_shape[1] * patches_shape[2]
  patches = tf.reshape(patches, shape=[patches.shape[0], patches.shape[1] * patches.shape[2], patches.shape[3]])

  def gather_random(p):
    random_indices = tf.random_uniform([num_patches], minval=0, maxval=patches_hw, dtype=tf.int32)
    return tf.gather(p, random_indices)

  # A little bit hard to do random per batch, so I just do the same random patches for all batches.

  ret = tf.map_fn(gather_random, patches, name='random_patches')
  ret = tf.reshape(ret, shape=[ret.shape[0] * ret.shape[1], ksizes[1], ksizes[2], image.shape[3]])
  return ret


#######################
# Miscellaneous Utils #
####################### 

def extract_patches(self):
        image_patches = tf.extract_image_patches(self.input_tensor, ksizes=[1, self.kernel_size,self.kernel_size, 1], strides=[1, self.stride_size,self.stride_size, 1], rates=[1, 1, 1, 1], padding=self.pad)
        return tf.reshape(image_patches, [self.in_N, self.Hout,self.Wout, self.kernel_size,self.kernel_size, self.in_depth]) 

def add_blockwise_grouplasso(t, block_row_size, block_col_size):
  raise NotImplementedError('Not debugged. And the implementation is very slow when block is small.')
  with tf.name_scope("BlockGroupLasso"):
    t = tf.expand_dims(tf.expand_dims(t,0),-1)
    blocks = tf.extract_image_patches(t,
                ksizes=[1, block_row_size, block_col_size, 1],
                strides=[1, block_row_size, block_col_size, 1],
                rates=[1, 1, 1, 1],
                padding='VALID')
    reg_sum = tf.constant(0.0)
    zero_blocks = 0.0
    total_blocks = 0.0
    blocks = tf.unstack(blocks) # list of 3-D tensors
    for b in blocks: # for each 3-D tensor
      for bb in tf.unstack(b): # for each 2-D tensor
        for block in tf.unstack(bb): # for each block
          blk_len = tf.sqrt(tf.reduce_sum(tf.square(block))) + tf.constant(1.0e-8)
          reg_sum = reg_sum + tf.cond(blk_len < zero_threshold,
                                      lambda: tf.constant(0.0),
                                      lambda: blk_len)

          # set them to zeros and calculate sparsity
          #block = tf.assign(block, tf.cond(blk_len < zero_threshold,
          #                                 lambda: tf.zeros_like(block),
          #                                 lambda: block))
          zero_blocks = zero_blocks + tf.cond( tf.equal(tf.reduce_sum(tf.square(block)), 0.0),
                                      lambda: tf.constant(1.0),
                                      lambda: tf.constant(0.0))
          total_blocks = total_blocks + 1.0
    return reg_sum, zero_blocks/total_blocks 

def add_blockwise_grouplasso(t, block_row_size, block_col_size):
  raise NotImplementedError('Not debugged. And the implementation is very slow when block is small.')
  with tf.name_scope("BlockGroupLasso"):
    t = tf.expand_dims(tf.expand_dims(t,0),-1)
    blocks = tf.extract_image_patches(t,
                ksizes=[1, block_row_size, block_col_size, 1],
                strides=[1, block_row_size, block_col_size, 1],
                rates=[1, 1, 1, 1],
                padding='VALID')
    reg_sum = tf.constant(0.0)
    zero_blocks = 0.0
    total_blocks = 0.0
    blocks = tf.unstack(blocks) # list of 3-D tensors
    for b in blocks: # for each 3-D tensor
      for bb in tf.unstack(b): # for each 2-D tensor
        for block in tf.unstack(bb): # for each block
          blk_len = tf.sqrt(tf.reduce_sum(tf.square(block))) + tf.constant(1.0e-8)
          reg_sum = reg_sum + tf.cond(blk_len < zero_threshold,
                                      lambda: tf.constant(0.0),
                                      lambda: blk_len)

          # set them to zeros and calculate sparsity
          #block = tf.assign(block, tf.cond(blk_len < zero_threshold,
          #                                 lambda: tf.zeros_like(block),
          #                                 lambda: block))
          zero_blocks = zero_blocks + tf.cond( tf.equal(tf.reduce_sum(tf.square(block)), 0.0),
                                      lambda: tf.constant(1.0),
                                      lambda: tf.constant(0.0))
          total_blocks = total_blocks + 1.0
    return reg_sum, zero_blocks/total_blocks 

def add_blockwise_grouplasso(t, block_row_size, block_col_size):
  raise NotImplementedError('Not debugged. And the implementation is very slow when block is small.')
  with tf.name_scope("BlockGroupLasso"):
    t = tf.expand_dims(tf.expand_dims(t,0),-1)
    blocks = tf.extract_image_patches(t,
                ksizes=[1, block_row_size, block_col_size, 1],
                strides=[1, block_row_size, block_col_size, 1],
                rates=[1, 1, 1, 1],
                padding='VALID')
    reg_sum = tf.constant(0.0)
    zero_blocks = 0.0
    total_blocks = 0.0
    blocks = tf.unstack(blocks) # list of 3-D tensors
    for b in blocks: # for each 3-D tensor
      for bb in tf.unstack(b): # for each 2-D tensor
        for block in tf.unstack(bb): # for each block
          blk_len = tf.sqrt(tf.reduce_sum(tf.square(block))) + tf.constant(1.0e-8)
          reg_sum = reg_sum + tf.cond(blk_len < zero_threshold,
                                      lambda: tf.constant(0.0),
                                      lambda: blk_len)

          # set them to zeros and calculate sparsity
          #block = tf.assign(block, tf.cond(blk_len < zero_threshold,
          #                                 lambda: tf.zeros_like(block),
          #                                 lambda: block))
          zero_blocks = zero_blocks + tf.cond( tf.equal(tf.reduce_sum(tf.square(block)), 0.0),
                                      lambda: tf.constant(1.0),
                                      lambda: tf.constant(0.0))
          total_blocks = total_blocks + 1.0
    return reg_sum, zero_blocks/total_blocks 

def conv(self, x, kernel_size, num_outputs, name, stride=1, padding="SAME", bias=True, std=1.0):
        assert len(x.get_shape()) == 5 # Policies x Batch x Height x Width x Feature
        with tf.variable_scope(name):
            w = self.create_weight_variable('w', std=std,
                                            shape=(kernel_size, kernel_size, int(x.get_shape()[-1].value), num_outputs))
            w = tf.reshape(w, [-1, kernel_size *kernel_size * int(x.get_shape()[-1].value), num_outputs])

            x_reshape = tf.reshape(x, (-1, x.get_shape()[2], x.get_shape()[3], x.get_shape()[4]))
            patches = tf.extract_image_patches(x_reshape, [1, kernel_size, kernel_size, 1], [1, stride, stride, 1], rates=[1, 1, 1, 1], padding=padding)
            final_shape = (tf.shape(x)[0], tf.shape(x)[1], patches.get_shape()[1].value, patches.get_shape()[2].value, num_outputs)
            patches = tf.reshape(patches, [tf.shape(x)[0],
                                           -1,
                                           kernel_size * kernel_size * x.get_shape()[-1].value])

            if self.indices is None:
                ret = tf.matmul(patches, w)
            else:
                ret = indexed_matmul(patches, w, self.indices)
            ret = tf.reshape(ret, final_shape)
            self.description += "Convolution layer {} with input shape {} and output shape {}\n".format(name, x.get_shape(), ret.get_shape())


            if bias:
                b = self.create_bias_variable('b', (1, 1, 1, num_outputs))
                if self.indices is not None:
                    b = tf.gather(b, self.indices)

                ret =  ret + b
            return ret 

def extract_patches(vgg_features, patch_size=1):
  patches_as_depth_vectors = tf.extract_image_patches(images=vgg_features,
                                                      ksizes=[1, patch_size, patch_size, 1],
                                                      strides=[1, 1, 1, 1], rates=[1, 1, 1, 1],
                                                      padding='VALID',
                                                      name='patches_as_depth_vectors')
  
  patches_NHWC = tf.reshape(patches_as_depth_vectors,
                            shape=[-1, patch_size, patch_size,
                                   patches_as_depth_vectors.shape[3].value],
                            name='patches_PHWC')
  
  patches_HWCN = tf.transpose(patches_NHWC, perm=[1, 2, 3, 0], name='patches_HWCP')
  
  return patches_HWCN 

def ternary_loss(im1, im2_warped, mask, max_distance=1):
    patch_size = 2 * max_distance + 1
    with tf.variable_scope('ternary_loss'):
        def _ternary_transform(image):
            intensities = tf.image.rgb_to_grayscale(image) * 255
            #patches = tf.extract_image_patches( # fix rows_in is None
            #    intensities,
            #    ksizes=[1, patch_size, patch_size, 1],
            #    strides=[1, 1, 1, 1],
            #    rates=[1, 1, 1, 1],
            #    padding='SAME')
            out_channels = patch_size * patch_size
            w = np.eye(out_channels).reshape((patch_size, patch_size, 1, out_channels))
            weights =  tf.constant(w, dtype=tf.float32)
            patches = tf.nn.conv2d(intensities, weights, strides=[1, 1, 1, 1], padding='SAME')

            transf = patches - intensities
            transf_norm = transf / tf.sqrt(0.81 + tf.square(transf))
            return transf_norm

        def _hamming_distance(t1, t2):
            dist = tf.square(t1 - t2)
            dist_norm = dist / (0.1 + dist)
            dist_sum = tf.reduce_sum(dist_norm, 3, keep_dims=True)
            return dist_sum

        t1 = _ternary_transform(im1)
        t2 = _ternary_transform(im2_warped)
        dist = _hamming_distance(t1, t2)

        transform_mask = create_mask(mask, [[max_distance, max_distance],
                                            [max_distance, max_distance]])
        return charbonnier_loss(dist, mask * transform_mask) 

def conv2d(inputs, weights, filter_shape, batch_norm, is_training,
           particles=1, strides=(1, 1, 1, 1), padding="SAME"):
    filter_height, filter_width, in_channels, out_channels = filter_shape

    def f1():
        weight = tf.reshape(weights[0, :-1, :], filter_shape)
        bias = tf.squeeze(weights[0, -1, :])
        conv = tf.nn.conv2d(inputs, filter=weight,
                            strides=strides, padding=padding)
        return tf.nn.bias_add(conv, bias)

    def f2():
        patches = tf.extract_image_patches(
            inputs,
            ksizes=[1, filter_height, filter_width, 1],
            strides=strides,
            rates=[1, 1, 1, 1],
            padding=padding)

        patches = _append_homog(patches)
        pb, h_out, w_out, flatten_size = patches.shape.as_list()
        patches = tf.reshape(patches, [particles, -1, flatten_size])
        preactivations = tf.matmul(patches, weights)
        preactivations = tf.reshape(preactivations, [-1, h_out, w_out, out_channels])
        return preactivations

    preactivations = tf.cond(tf.equal(particles, 1), f1, f2)
    if batch_norm:
        bn = tf.layers.batch_normalization(preactivations, training=is_training)
        activations = tf.nn.relu(bn)
    else:
        activations = tf.nn.relu(preactivations)
    return preactivations, activations 

def extract_random_patches(image, patch_sizes, num_patches):
  """Extracts random patches from an image tensor."""
  if isinstance(patch_sizes, int):
    ksizes = [1, patch_sizes, patch_sizes, 1]
  elif isinstance(patch_sizes, list) or isinstance(patch_sizes, tuple):
    if len(patch_sizes) == 2:
      ksizes = [1, patch_sizes[0], patch_sizes[1], 1]
    elif len(patch_sizes) == 4:
      ksizes = patch_sizes
    else:
      raise ValueError('patch_sizes must be length 2 or length 4.')
  else:
    raise ValueError('patch_sizes must be a length 2 or length 4 list, or an int.')

  # (batch, height, width, patch_size * patch_size * feature)
  patches = tf.extract_image_patches(image, ksizes=ksizes,
                                     strides=[1, 1, 1, 1], rates=[1, 1, 1, 1], padding='VALID')
  patches_shape = tf.unstack(tf.shape(patches))
  patches_hw = patches_shape[1] * patches_shape[2]
  patches = tf.reshape(patches, shape=[patches.shape[0], patches.shape[1] * patches.shape[2], patches.shape[3]])

  def gather_random(p):
    random_indices = tf.random_uniform([num_patches], minval=0, maxval=patches_hw, dtype=tf.int32)
    return tf.gather(p, random_indices)

  # A little bit hard to do random per batch, so I just do the same random patches for all batches.

  ret = tf.map_fn(gather_random, patches, name='random_patches')
  ret = tf.reshape(ret, shape=[ret.shape[0] * ret.shape[1], ksizes[1], ksizes[2], image.shape[3]])
  return ret


#######################
# Miscellaneous Utils #
####################### 

def census_loss(self, img1, img2_warped, mask, max_distance=3):
        patch_size = 2 * max_distance + 1
        with tf.variable_scope('census_loss'):
            def _ternary_transform(image):
                intensities = tf.image.rgb_to_grayscale(image) * 255
                #patches = tf.extract_image_patches( # fix rows_in is None
                #    intensities,
                #    ksizes=[1, patch_size, patch_size, 1],
                #    strides=[1, 1, 1, 1],
                #    rates=[1, 1, 1, 1],
                #    padding='SAME')
                out_channels = patch_size * patch_size
                w = np.eye(out_channels).reshape((patch_size, patch_size, 1, out_channels))
                weights =  tf.constant(w, dtype=tf.float32)
                patches = tf.nn.conv2d(intensities, weights, strides=[1, 1, 1, 1], padding='SAME')
    
                transf = patches - intensities
                transf_norm = transf / tf.sqrt(0.81 + tf.square(transf))
                return transf_norm
    
            def _hamming_distance(t1, t2):
                dist = tf.square(t1 - t2)
                dist_norm = dist / (0.1 + dist)
                dist_sum = tf.reduce_sum(dist_norm, 3, keepdims=True)
                return dist_sum
    
            t1 = _ternary_transform(img1)
            t2 = _ternary_transform(img2_warped)
            dist = _hamming_distance(t1, t2)
    
            transform_mask = self.create_mask(mask, [[max_distance, max_distance],
                                                [max_distance, max_distance]])
            return self.abs_robust_loss(dist, mask * transform_mask) 

def create_res_hessian_computing_tf_graph(input_shape, layer_kernel, layer_stride):
    """ 
    This function create the TensorFlow graph for computing hessian matrix for res layer.
    Step 1: It first extract image patches using tf.extract_image_patches.
    Step 2: Then calculate the hessian matrix by outer product.

    Args:
        input_shape: the dimension of input
        layer_kernel: kernel size of the layer
        layer_stride: stride of the layer
    Output:
        input_holder: TensorFlow placeholder for layer input
        get_hessian_op: A TensorFlow operator to calculate hessian matrix

    """ 
    input_holder = tf.placeholder(dtype=tf.float32, shape=input_shape)
    patches = tf.extract_image_patches(images = input_holder,
                                       ksizes = [1,layer_kernel, layer_kernel,1],
                                       strides = [1, layer_stride, layer_stride, 1],
                                       rates = [1, 1, 1, 1],
                                       padding = 'SAME')
    print 'Patches shape: %s' %patches.get_shape()
    a = tf.expand_dims(patches, axis=-1)
    b = tf.expand_dims(patches, axis=3)
    outprod = tf.multiply(a, b)
    # print 'outprod shape: %s' %outprod.get_shape()
    get_hessian_op = tf.reduce_mean(outprod, axis=[0, 1, 2])
    print 'Hessian shape: %s' % get_hessian_op.get_shape()
    return input_holder, get_hessian_op


# Construct hessian computing graph for fc layer 

def im2col(
    x: Union[tf.Tensor, np.ndarray],
    h_filter: int,
    w_filter: int,
    padding: str,
    stride: int,
) -> tf.Tensor:
    """Generic implementation of im2col on tf.Tensors."""

    with tf.name_scope("im2col"):

        # we need NHWC because tf.extract_image_patches expects this
        nhwc_tensor = tf.transpose(x, [0, 2, 3, 1])
        channels = int(nhwc_tensor.shape[3])

        # extract patches
        patch_tensor = tf.extract_image_patches(
            nhwc_tensor,
            ksizes=[1, h_filter, w_filter, 1],
            strides=[1, stride, stride, 1],
            rates=[1, 1, 1, 1],
            padding=padding,
        )

        # change back to NCHW
        patch_tensor_nchw = tf.reshape(
            tf.transpose(patch_tensor, [3, 1, 2, 0]), (h_filter, w_filter, channels, -1)
        )

        # reshape to x_col
        x_col_tensor = tf.reshape(
            tf.transpose(patch_tensor_nchw, [2, 0, 1, 3]),
            (channels * h_filter * w_filter, -1),
        )

        return x_col_tensor 

def make_covariance_update_op(self, ema_decay, ema_weight):
    filter_height, filter_width, _, _ = self._filter_shape

    # TODO(b/64144716): there is potential here for a big savings in terms
    # of memory use.
    if self._dilations is None:
      rates = (1, 1, 1, 1)
    else:
      rates = tuple(self._dilations)

    self._patches = []
    for tower in range(self._num_towers):
      with maybe_place_on_device(self._get_data_device(tower)):
        patches = tf.extract_image_patches(
            self._inputs[tower],
            ksizes=[1, filter_height, filter_width, 1],
            strides=self._strides,
            rates=rates,
            padding=self._padding)

        if self._patch_mask is not None:
          assert self._patch_mask.shape == self._filter_shape[0:-1]
          # This should work as intended due to broadcasting.
          patches *= self._patch_mask

        if self._has_bias:
          patches = append_homog(patches)

        self._patches.append(patches)

    return super(ConvDiagonalFactor, self).make_covariance_update_op(
        ema_decay, ema_weight) 

def reorg_layer(net, stride=2, name='reorg'):
    with tf.name_scope(name):
        net = tf.extract_image_patches(net,
                                       ksizes=[1, stride, stride, 1],
                                       strides=[1, stride, stride, 1],
                                       rates=[1, 1, 1, 1],
                                       padding='VALID')
    return net 

def extract_image_patches(x, ksizes, ssizes, padding='same',
                          data_format='channels_last'):
    '''
    Extract the patches from an image
    # Parameters
        x : The input image
        ksizes : 2-d tuple with the kernel size
        ssizes : 2-d tuple with the strides size
        padding : 'same' or 'valid'
        data_format : 'channels_last' or 'channels_first'
    # Returns
        The (k_w,k_h) patches extracted
        TF ==> (batch_size,w,h,k_w,k_h,c)
        TH ==> (batch_size,w,h,c,k_w,k_h)
    '''
    kernel = [1, ksizes[0], ksizes[1], 1]
    strides = [1, ssizes[0], ssizes[1], 1]
    padding = _preprocess_padding(padding)
    if data_format == 'channels_first':
        x = KTF.permute_dimensions(x, (0, 2, 3, 1))
    bs_i, w_i, h_i, ch_i = KTF.int_shape(x)
    patches = tf.extract_image_patches(x, kernel, strides, [1, 1, 1, 1],
                                       padding)
    # Reshaping to fit Theano
    bs, w, h, ch = KTF.int_shape(patches)
    patches = tf.reshape(tf.transpose(tf.reshape(patches, [-1, w, h, tf.floordiv(ch, ch_i), ch_i]), [0, 1, 2, 4, 3]),
                         [-1, w, h, ch_i, ksizes[0], ksizes[1]])
    if data_format == 'channels_last':
        patches = KTF.permute_dimensions(patches, [0, 1, 2, 4, 5, 3])
    return patches 

def _reorg(self, name, inputs, stride, data_format, use_space_to_depth=True, darknet_original=False):
        with tf.name_scope(name):
            # darknet original reorg layer is weird.
            # As default we don't use darknet original reorg.
            # See detail: https://github.com/LeapMind/lmnet/issues/16
            if darknet_original:
                # TODO(wakisaka): You can use only data_format == "NHWC", yet.
                assert data_format == "NHWC"
                input_shape = tf.shape(inputs)
                # channel shape use static.
                b, h, w, c = input_shape[0], input_shape[1], input_shape[2], inputs.get_shape().as_list()[3]
                output_h = h // stride
                output_w = w // stride
                output_c = c * stride * stride
                transpose_tensor = tf.transpose(inputs, [0, 3, 1, 2])
                reshape_tensor = tf.reshape(transpose_tensor, [b, (c // (stride * stride)), h, stride, w, stride])
                transpose_tensor = tf.transpose(reshape_tensor, [0, 3, 5, 1, 2, 4])
                reshape_tensor = tf.reshape(transpose_tensor, [b, output_c, output_h, output_w])
                transpose_tensor = tf.transpose(reshape_tensor, [0, 2, 3, 1])
                outputs = tf.reshape(transpose_tensor, [b, output_h, output_w, output_c])

                return outputs
            else:
                # tf.extract_image_patches() raise error with images_placeholder `None` shape as dynamic image.
                # Github issue: https://github.com/leapmindadmin/lmnet/issues/17
                # Currently, I didn't try to space_to_depth with images_placeholder `None` shape as dynamic image.
                if use_space_to_depth:

                    outputs = tf.space_to_depth(inputs, stride, data_format=data_format)
                    return outputs

                else:
                    # TODO(wakisaka): You can use only data_format == "NHWC", yet.
                    assert data_format == "NHWC"
                    ksize = [1, stride, stride, 1]
                    strides = [1, stride, stride, 1]
                    rates = [1, 1, 1, 1]
                    outputs = tf.extract_image_patches(inputs, ksize, strides, rates, "VALID")

                    return outputs 

def forward(self):
        inp = self.inp.out
        s = self.lay.stride
        self.out = tf.extract_image_patches(
            inp, [1,s,s,1], [1,s,s,1], [1,1,1,1], 'VALID') 

def forward(self):
        inp = self.inp.out
        s = self.lay.stride
        self.out = tf.extract_image_patches(
            inp, [1,s,s,1], [1,s,s,1], [1,1,1,1], 'VALID') 

def testGradient(self):
    # Set graph seed for determinism.
    random_seed = 42
    tf.set_random_seed(random_seed)

    with self.test_session():
      for test_case in self._TEST_CASES:
        np.random.seed(random_seed)
        in_shape = test_case['in_shape']
        in_val = tf.constant(np.random.random(in_shape),
                             dtype=tf.float32)

        for padding in ['VALID', 'SAME']:
          out_val = tf.extract_image_patches(in_val,
                                             test_case['ksizes'],
                                             test_case['strides'],
                                             test_case['rates'],
                                             padding)
          out_shape = out_val.get_shape().as_list()

          err = tf.test.compute_gradient_error(
              in_val, in_shape, out_val, out_shape
          )

          print('extract_image_patches gradient err: %.4e' % err)
          self.assertLess(err, 1e-4)