Python torch.gt() Examples

def score_msk_ent(ner_mat, ner_dict):
    seq_len, batch_sz = ner_mat.size()
    assert ner_dict.fword2idx('O') == 0
    # msk = torch.zeros((batch_sz, seq_len))
    ner = ner_mat.transpose(1, 0)
    indicator = torch.gt(ner, 0).int()
    global_bag = []
    for bid in range(batch_sz):
        tmp_bag = []
        for t in range(seq_len):
            if indicator[bid][t] != -1:

                if indicator[bid][t] == 0:
                    indicator, l = rec(indicator, bid, t, indicator[bid][t], seq_len)
                    tmp_bag.append([0, l])
                else:
                    indicator, l = rec(indicator, bid, t, indicator[bid][t], seq_len)
                    tmp_bag.append([1, l])
        global_bag.append(tmp_bag)
    return global_bag 

def check_mask_rele(mask_ratio=0.4):
    mat = torch.randint(size=(1, 20), low=-2, high=3)

    mat = torch.squeeze(mat,dim=0)
    print('mat', mat.size(), mat)

    all_rele_inds = torch.gt(mat, torch_zero).nonzero()
    print('all_rele_inds', all_rele_inds.size(), all_rele_inds)
    num_rele = all_rele_inds.size()[0]
    print('num_rele', num_rele)

    num_to_mask = int(num_rele*mask_ratio)
    mask_inds = np.random.choice(num_rele, size=num_to_mask, replace=False)
    print('mask_inds', mask_inds)

    rele_inds_to_mask = all_rele_inds[mask_inds, 0]
    print('rele_inds_to_mask', rele_inds_to_mask) 

def get_score(self, model, texta, textb, labels, score_type='f1'):
        metrics_map = {
            'f1': f1_score,
            'p': precision_score,
            'r': recall_score,
            'acc': accuracy_score
        }
        metric_func = metrics_map[score_type] if score_type in metrics_map else metrics_map['f1']
        assert texta.size(1) == textb.size(1) == len(labels)
        predict_prob = model(texta, textb)
        # print('predict', predict_prob)
        # print('labels', labels)
        predict_labels = torch.gt(predict_prob, 0.5)
        predict_labels = predict_labels.view(-1).cpu().data.numpy()
        labels = labels.view(-1).cpu().data.numpy()
        return metric_func(predict_labels, labels, average='micro') 

def calc_mask(self, sparsity, wrapper, wrapper_idx=None):
        assert wrapper.type == 'BatchNorm2d', 'SlimPruner only supports 2d batch normalization layer pruning'
        weight = wrapper.module.weight.data.clone()
        if wrapper.weight_mask is not None:
            # apply base mask for iterative pruning
            weight = weight * wrapper.weight_mask

        base_mask = torch.ones(weight.size()).type_as(weight).detach()
        mask = {'weight_mask': base_mask.detach(), 'bias_mask': base_mask.clone().detach()}
        filters = weight.size(0)
        num_prune = int(filters * sparsity)
        if filters >= 2 and num_prune >= 1:
            w_abs = weight.abs()
            mask_weight = torch.gt(w_abs, self.global_threshold).type_as(weight)
            mask_bias = mask_weight.clone()
            mask = {'weight_mask': mask_weight.detach(), 'bias_mask': mask_bias.detach()}
        return mask 

def train_generator(self, train_data=None, generated_data=None, generator=None, discriminator=None, **kwargs):
        for entry in train_data:
            qid, batch_ranking, batch_label = entry[0], entry[1], entry[2]
            if gpu: batch_ranking = batch_ranking.to(device)

            pos_inds = torch.gt(torch.squeeze(batch_label), 0).nonzero()

            g_preds = generator.predict(batch_ranking, train=True)
            g_probs = torch.sigmoid(torch.squeeze(g_preds))

            neg_inds = torch.multinomial(g_probs, pos_inds.size(0), replacement=True)

            pos_docs = batch_ranking[:, pos_inds[:, 0], :]
            neg_docs = batch_ranking[:, neg_inds, :]

            reward = discriminator.get_reward(pos_docs=pos_docs, neg_docs=neg_docs, loss_type=self.loss_type)

            g_loss = -torch.mean((torch.log(g_probs[neg_inds]) * reward))

            generator.optimizer.zero_grad()
            g_loss.backward()
            generator.optimizer.step() 

def sample_sigmoid(args, y, sample=False):
    r"""
    Sample from scores between 0 and 1 as means of Bernouolli distribution, or threshold over 0.5
    
    :param args: parsed arguments
    :param y: values to threshold
    :param sample: if True, sample, otherwise, threshold
    :return: sampled/thresholed values, in {0., 1.}
    """
    thresh = 0.5
    if sample:
        y_thresh = torch.rand(y.size(0), y.size(1), y.size(2)).to(args.device)
        y_result = torch.gt(y, y_thresh).float()
    else:
        y_thresh = (torch.ones(y.size(0), y.size(1), y.size(2)) * thresh).to(args.device)
        y_result = torch.gt(y, y_thresh).float()
    return y_result 

def check_monitor_top_k(self, current):
        less_than_k_models = len(self.best_k_models) < self.save_top_k
        if less_than_k_models:
            return True

        if not isinstance(current, torch.Tensor):
            rank_zero_warn(
                f'{current} is supposed to be a `torch.Tensor`. Saving checkpoint may not work correctly.'
                f' HINT: check the value of {self.monitor} in your validation loop', RuntimeWarning
            )
            current = torch.tensor(current)

        monitor_op = {
            "min": torch.lt,
            "max": torch.gt,
        }[self.mode]

        return monitor_op(current, self.best_k_models[self.kth_best_model_path]) 

def _graph_fn_get_action_components(self, flat_key, logits, parameters, deterministic):
        action_space_component = self.flat_action_space[flat_key]

        # Skip our distribution, iff discrete action-space and deterministic acting (greedy).
        # In that case, one does not need to create a distribution in the graph each act (only to get the argmax
        # over the logits, which is the same as the argmax over the probabilities (or log-probabilities)).
        if isinstance(action_space_component, IntBox) and \
                (deterministic is True or (isinstance(deterministic, np.ndarray) and deterministic)):
            return self._graph_fn_get_deterministic_action_wo_distribution(logits)
        # Bernoulli: Sigmoid derived p must be larger 0.5.
        elif isinstance(action_space_component, BoolBox) and \
                (deterministic is True or (isinstance(deterministic, np.ndarray) and deterministic)):
            # Note: Change 0.5 to 1.0, once parameters are logits, not probs anymore (so far, parameters for
            # Bernoulli distributions are still probs).
            if get_backend() == "tf":
                return tf.greater(parameters, 0.5)
            elif get_backend() == "pytorch":
                return torch.gt(parameters, 0.5)
        # Deterministic is tensor or False. Pass through graph.
        else:
            return self.distributions[flat_key].draw(parameters, deterministic) 

def _make_masked_tokens(sents, pad_idx):
    lengths = [len(sent) for sent in sents]
    max_len = max(lengths)
    bsz = len(lengths)
    # Tensor containing the (right) padded tokens
    tokens = th.full((max_len, bsz), pad_idx).long()
    for i in range(bsz):
        tokens[:lengths[i], i] = th.LongTensor(sents[i])
    # Mask such that mask[i, b] = 1 iff lengths[b] < i
    lengths = th.LongTensor(lengths).view(1, -1)
    mask = th.gt(th.arange(max_len).view(-1, 1), lengths)

    return tokens, mask 

def forward(self, inputs, mask):
        """
        Args:
            inputs (FloatTensor): ``(batch_size, src_len, model_dim)``
            mask (LongTensor): ``(batch_size, src_len, src_len)``

        Returns:
            (FloatTensor):

            * outputs ``(batch_size, src_len, model_dim)``
        """
        dec_mask = None
        src_len = mask.size(-1)
        future_mask = torch.ones(
            [src_len, src_len],
            device=mask.device,
            dtype=torch.uint8)
        future_mask = future_mask.triu_(1).view(1, src_len, src_len)
        dec_mask = torch.gt(mask + future_mask, 0)

        input_norm = self.layer_norm_1(inputs)
        context, _ = self.self_attn(input_norm, input_norm, input_norm,
                                    mask=dec_mask, type="self")

        context = self.dropout(context) + inputs
        context_norm = self.layer_norm_2(context)

        output = self.feed_forward(context_norm)
        output = output + context

        return output 

def threshold_policy(self, weights, thresholds, threshold_criteria, dim=1):
        """
        """
        if threshold_criteria == 'Mean_Abs':
            return weights.data.abs().mean(dim=dim).gt(thresholds).type(weights.type())
        elif threshold_criteria == 'Max':
            maxv, _ = weights.data.abs().max(dim=dim)
            return maxv.gt(thresholds).type(weights.type())
        exit("Invalid threshold_criteria {}".format(threshold_criteria)) 

def per_query_generation(self, qid, batch_ranking, batch_label, generator):

        used_batch_label = batch_label

        if gpu: batch_ranking = batch_ranking.to(device)

        # [1, ranking_size] -> [ranking_size]
        # [z, n] If input has n dimensions, then the resulting indices tensor out is of size (z×n), where z is the total number of non-zero elements in the input tensor.
        all_pos_inds = torch.gt(torch.squeeze(used_batch_label), 0).nonzero()
        num_pos = all_pos_inds.size()[0]

        if num_pos < 1:
            return None

        ranking_size = batch_label.size(1)
        if num_pos < ranking_size:
            half = int(ranking_size*0.5)
            if num_pos<= half:
                num_samples = num_pos
                pos_inds = all_pos_inds[:, 0]
            else: # aiming for a balance
                num_samples = half
                pos_inds = all_pos_inds[0:half, 0]

            batch_pred = generator.predict(batch_ranking)  # [batch, size_ranking]
            batch_prob = F.softmax(torch.squeeze(batch_pred), dim=0)

            #print('num_samples', num_samples)
            #print('batch_prob', batch_prob)

            neg_inds = torch.multinomial(batch_prob, num_samples, replacement=True)

            # todo cpu inds & cuda inds w.r.t. other methods
            #if gpu: pos_inds = pos_inds.to(device)
            return (pos_inds, neg_inds)
        else:
            return None 

def positive_predictions_for(predicted_probs, threshold=0.5):
    #return sum(torch.gt(predicted_probs.data, threshold).cpu().numpy().tolist())
    return (torch.gt(predicted_probs.data, threshold).float().sum()) 

def forward(
        self, inputs, memory_bank, src_pad_mask, tgt_pad_mask, previous_input=None, layer_cache=None, step=None,
    ):
        """
        Args:
            inputs (`FloatTensor`): `[batch_size x 1 x model_dim]`
            memory_bank (`FloatTensor`): `[batch_size x src_len x model_dim]`
            src_pad_mask (`LongTensor`): `[batch_size x 1 x src_len]`
            tgt_pad_mask (`LongTensor`): `[batch_size x 1 x 1]`

        Returns:
            (`FloatTensor`, `FloatTensor`, `FloatTensor`):

            * output `[batch_size x 1 x model_dim]`
            * attn `[batch_size x 1 x src_len]`
            * all_input `[batch_size x current_step x model_dim]`

        """
        dec_mask = torch.gt(tgt_pad_mask + self.mask[:, : tgt_pad_mask.size(1), : tgt_pad_mask.size(1)], 0)
        input_norm = self.layer_norm_1(inputs)
        all_input = input_norm
        if previous_input is not None:
            all_input = torch.cat((previous_input, input_norm), dim=1)
            dec_mask = None

        query = self.self_attn(all_input, all_input, input_norm, mask=dec_mask, layer_cache=layer_cache, type="self",)

        query = self.drop(query) + inputs

        query_norm = self.layer_norm_2(query)
        mid = self.context_attn(
            memory_bank, memory_bank, query_norm, mask=src_pad_mask, layer_cache=layer_cache, type="context",
        )
        output = self.feed_forward(self.drop(mid) + query)

        return output, all_input
        # return output 

def __init__(self, monitor: str = 'val_loss', min_delta: float = 0.0, patience: int = 3,
                 verbose: bool = False, mode: str = 'auto', strict: bool = True):
        super().__init__()
        self.monitor = monitor
        self.patience = patience
        self.verbose = verbose
        self.strict = strict
        self.min_delta = min_delta
        self.wait_count = 0
        self.stopped_epoch = 0
        self.mode = mode

        if mode not in self.mode_dict:
            if self.verbose > 0:
                log.info(f'EarlyStopping mode {mode} is unknown, fallback to auto mode.')
            self.mode = 'auto'

        if self.mode == 'auto':
            if self.monitor == 'acc':
                self.mode = 'max'
            else:
                self.mode = 'min'
            if self.verbose > 0:
                log.info(f'EarlyStopping mode set to {self.mode} for monitoring {self.monitor}.')

        self.min_delta *= 1 if self.monitor_op == torch.gt else -1
        self.best_score = torch_inf if self.monitor_op == torch.lt else -torch_inf 

def _compute_top_k(self, features):
        """
         Args:
            features: prediction matrix (before softmax) with shape (batch_size, feature_dim)
         Return: 
            top_k largest distances
        """
        # reading the codes below can help understand better
        '''
        dist_array_2 = self._pairwise_distance(features)
        n = features.size(0)
        mask = torch.zeros(n, n)
        if self.use_gpu: mask=mask.cuda()
        for i in range(0, n):
            for j in range(i+1, n):
                mask[i, j] += 1
        dist_array_2 = dist_array_2 * mask
        dist_array_2 = dist_array_2.view(1, -1)
        dist_array_2 = dist_array_2[torch.gt(dist_array_2, 0)]
        top_k_2 = dist_array_2.sort()[0][-self.k:]
        print(top_k_2)
        '''
        dist_array = self._pairwise_distance(features)
        dist_array = dist_array.view(1, -1)
        top_k = dist_array.sort()[0][0, -self.k * 2::2]     # Because there are 2 same value of same feature pair in the dist_array
        # print('top k intra class dist:', top_k)
        return top_k 

def _compute_min_dist(self, center_features):
        """
         Args:
            center_features: center matrix (before softmax) with shape (center_number, center_dim)
         Return: 
            minimum center distance
        """
        '''
        # reading codes below can help understand better
        dist_array = self._pairwise_distance(center_features)
        n = center_features.size(0)
        mask = torch.zeros(n, n)
        if self.use_gpu: mask=mask.cuda()
        for i in range(0, n):
            for j in range(i + 1, n):
                mask[i, j] += 1
        dist_array *= mask
        dist_array = dist_array.view(1, -1)
        dist_array = dist_array[torch.gt(dist_array, 0)]
        min_inter_class_dist = dist_array.min()
        print(min_inter_class_dist)
        '''
        n = center_features.size(0)
        dist_array2 = self._pairwise_distance(center_features)
        min_inter_class_dist2 = dist_array2.view(1, -1).sort()[0][0][n]  # exclude self compare, the first one is the min_inter_class_dist
        return min_inter_class_dist2 

def _projection(self, weight, sparsity):
        '''
        Return the Euclidean projection of the weight matrix according to the pruning mode.

        Parameters
        ----------
        weight : tensor
            original matrix
        sparsity : float
            the ratio of parameters which need to be set to zero

        Returns
        -------
        tensor
            the projected matrix
        '''
        w_abs = weight.abs()
        if self._base_algo == 'level':
            k = int(weight.numel() * sparsity)
            if k == 0:
                mask_weight = torch.ones(weight.shape).type_as(weight)
            else:
                threshold = torch.topk(w_abs.view(-1), k, largest=False)[0].max()
                mask_weight = torch.gt(w_abs, threshold).type_as(weight)
        elif self._base_algo in ['l1', 'l2']:
            filters = weight.size(0)
            num_prune = int(filters * sparsity)
            if filters < 2 or num_prune < 1:
                mask_weight = torch.ones(weight.size()).type_as(weight).detach()
            else:
                w_abs_structured = w_abs.view(filters, -1).sum(dim=1)
                threshold = torch.topk(w_abs_structured.view(-1), num_prune, largest=False)[0].max()
                mask_weight = torch.gt(w_abs_structured, threshold)[:, None, None, None].expand_as(weight).type_as(weight)

        return weight.data.mul(mask_weight) 

def get_mask(self, base_mask, weight, num_prune, wrapper, wrapper_idx):
        filters = weight.shape[0]
        w_abs = weight.abs()
        w_abs_structured = w_abs.view(filters, -1).sum(dim=1)
        threshold = torch.topk(w_abs_structured.view(-1), num_prune, largest=False)[0].max()
        mask_weight = torch.gt(w_abs_structured, threshold)[:, None, None, None].expand_as(weight).type_as(weight)
        mask_bias = torch.gt(w_abs_structured, threshold).type_as(weight).detach() if base_mask['bias_mask'] is not None else None

        return {'weight_mask': mask_weight.detach(), 'bias_mask': mask_bias} 

def random_mask_rele_labels(batch_ranking, batch_label=None, mask_ratio=None, mask_value=0, presort=False):
    '''
    Mask the ground-truth labels with the specified ratio as '0'. Meanwhile, re-sort according to the labels if required.
    :param doc_reprs:
    :param doc_labels:
    :param mask_ratio: the ratio of labels to be masked
    :param mask_value:
    :param presort:
    :return:
    '''

    assert 1 == batch_label.size(0) # todo for larger batch-size, need to per-dimension masking

    # squeeze for easy process
    docs, labels = torch.squeeze(batch_ranking, dim=0), torch.squeeze(batch_label)

    all_rele_inds = torch.gt(labels, torch_zero).nonzero()
    num_rele = all_rele_inds.size()[0]

    num_to_mask = int(num_rele*mask_ratio)
    mask_inds = np.random.choice(num_rele, size=num_to_mask, replace=False)

    rele_inds_to_mask = all_rele_inds[mask_inds, 0] # the 0-column corresponds to original rele index since all_rele_inds.size()=(num_rele, 1)

    batch_label[:, rele_inds_to_mask] = mask_value

    if torch.gt(batch_label, torch_zero).any(): # whether the masked one includes explicit positive labels
        if presort: # re-sort according to the labels if required
            std_labels = torch.squeeze(batch_label)
            sorted_labels, sorted_inds = torch.sort(std_labels, descending=True)

            batch_label = torch.unsqueeze(sorted_labels, dim=0)
            batch_ranking = batch_ranking[:, sorted_inds, :]

        return batch_ranking, batch_label
    else:
        # only supports enough rele labels
        raise NotImplementedError 

def random_mask_all_labels(batch_ranking, batch_label, mask_ratio, mask_value=0, presort=False):
    '''
    Mask the ground-truth labels with the specified ratio as '0'. Meanwhile, re-sort according to the labels if required.
    :param doc_reprs:
    :param doc_labels:
    :param mask_ratio: the ratio of labels to be masked
    :param mask_value:
    :param presort:
    :return:
    '''

    size_ranking = batch_label.size(1)
    num_to_mask = int(size_ranking*mask_ratio)
    mask_ind = np.random.choice(size_ranking, size=num_to_mask, replace=False)

    batch_label[:, mask_ind] = mask_value

    if torch.gt(batch_label, torch_zero).any(): # whether the masked one includes explicit positive labels
        if presort: # re-sort according to the labels if required
            std_labels = torch.squeeze(batch_label)
            sorted_labels, sorted_inds = torch.sort(std_labels, descending=True)

            batch_label = torch.unsqueeze(sorted_labels, dim=0)
            batch_ranking = batch_ranking[:, sorted_inds, :]

        return batch_ranking, batch_label
    else:
        return None 

def threshold_mask(weights, threshold):
    """Create a threshold mask for the provided parameter tensor using
    magnitude thresholding.

    Arguments:
        weights: a parameter tensor which should be pruned.
        threshold: the pruning threshold.
    Returns:
        prune_mask: The pruning mask.
    """
    return torch.gt(torch.abs(weights), threshold).type(weights.type()) 

def get_weighted_clipped_pos_diffs(sorted_std_labels):
    #num_pos = torch.nonzero(sorted_std_labels).size(0)
    #print('sorted_std_labels', sorted_std_labels)
    num_pos = torch.gt(sorted_std_labels, 0).nonzero().size(0) # supporting the case of including '-1'

    #total_items = sorted_std_labels.size(0)
    total_items = torch.ge(sorted_std_labels, 0).nonzero().size(0)

    mat_diffs = torch.unsqueeze(sorted_std_labels, dim=1) - torch.unsqueeze(sorted_std_labels, dim=0)
    pos_diffs = torch.where(mat_diffs < 0, tor_zero, mat_diffs)
    clipped_pos_diffs = pos_diffs[0:num_pos, 0:total_items]
    #print('clipped_pos_diffs', clipped_pos_diffs)

    total_true_pairs = torch.nonzero(clipped_pos_diffs).size(0)

    r_discounts = torch.arange(total_items).type(tensor)
    r_discounts = torch.log2(2.0 + r_discounts)
    r_discounts = torch.unsqueeze(r_discounts, dim=0)

    c_discounts = torch.arange(num_pos).type(tensor)
    c_discounts = torch.log2(2.0 + c_discounts)
    c_discounts = torch.unsqueeze(c_discounts, dim=1)

    weighted_clipped_pos_diffs = clipped_pos_diffs / r_discounts
    weighted_clipped_pos_diffs = weighted_clipped_pos_diffs / c_discounts

    #print(weighted_clipped_pos_diffs)

    return weighted_clipped_pos_diffs, total_true_pairs, total_items 

def generate_true_docs(qid, truth_label, num_samples, dict_true_inds=None):
    '''
    :param qid:
    :param truth_label: it is fine if including '-1'
    :param num_samples:
    :param dict_true_inds:
    :return:
    '''
    if dict_true_inds is not None and qid in dict_true_inds:
        true_inds, size_unique = dict_true_inds[qid]

        if num_samples is None or num_samples>size_unique:
            num_samples = int(0.5*size_unique)
            if num_samples < 1:
                num_samples = 1

        rand_inds = torch.multinomial(torch.ones(size_unique), num_samples, replacement=False)
        return true_inds[rand_inds]
    else:
        # [z, n] If input has n dimensions, then the resulting indices tensor out is of size (z×n), where z is the total number of non-zero elements in the input tensor.
        true_inds = torch.gt(truth_label, 0).nonzero()

        size_unique = true_inds.size(0)
        true_inds = true_inds[:, 0]
        if dict_true_inds is not None: #buffer
            dict_true_inds[qid] = (true_inds, size_unique)

        if num_samples is None or num_samples>size_unique:
            num_samples = int(0.5*size_unique)
            if num_samples < 1:
                num_samples = 1

        rand_inds = torch.multinomial(torch.ones(size_unique), num_samples, replacement=False)
        return true_inds[rand_inds] 

def gt(t1, t2):
    """
    Element-wise rich greater than comparison between values from operand t1 with respect to values of
    operand t2 (i.e. t1 > t2), not commutative.
    Takes the first and second operand (scalar or tensor) whose elements are to be compared as argument.

    Parameters
    ----------
    t1: tensor or scalar
       The first operand to be compared greater than second operand

    t2: tensor or scalar
       The second operand to be compared less than first operand

    Returns
    -------
    result: ht.DNDarray
       A uint8-tensor holding 1 for all elements in which values of t1 are greater than values of t2,
       0 for all other elements

    Examples
    -------
    >>> import heat as ht
    >>> T1 = ht.float32([[1, 2],[3, 4]])
    >>> ht.gt(T1, 3.0)
    tensor([[0, 0],
            [0, 1]], dtype=torch.uint8)

    >>> T2 = ht.float32([[2, 2], [2, 2]])
    >>> ht.gt(T1, T2)
    tensor([[0, 0],
            [1, 1]], dtype=torch.uint8)
    """
    return operations.__binary_op(torch.gt, t1, t2) 

def _compute_min_dist(self, center_features):
        """
         Args:
            center_features: center matrix (before softmax) with shape (center_number, center_dim)
         Return: 
            minimum center distance
        """
        '''
        # reading codes below can help understand better
        dist_array = self._pairwise_distance(center_features)
        n = center_features.size(0)
        mask = torch.zeros(n, n)
        if self.use_gpu: mask=mask.cuda()
        for i in range(0, n):
            for j in range(i + 1, n):
                mask[i, j] += 1
        dist_array *= mask
        dist_array = dist_array.view(1, -1)
        dist_array = dist_array[torch.gt(dist_array, 0)]
        min_inter_class_dist = dist_array.min()
        print(min_inter_class_dist)
        '''
        n = center_features.size(0)
        dist_array2 = self._pairwise_distance(center_features)
        min_inter_class_dist2 = dist_array2.view(1, -1).sort()[0][0][n]  # exclude self compare, the first one is the min_inter_class_dist
        return min_inter_class_dist2 

def _compute_top_k(self, features):
        """
         Args:
            features: prediction matrix (before softmax) with shape (batch_size, feature_dim)
         Return: 
            top_k largest distances
        """
        # reading the codes below can help understand better
        '''
        dist_array_2 = self._pairwise_distance(features)
        n = features.size(0)
        mask = torch.zeros(n, n)
        if self.use_gpu: mask=mask.cuda()
        for i in range(0, n):
            for j in range(i+1, n):
                mask[i, j] += 1
        dist_array_2 = dist_array_2 * mask
        dist_array_2 = dist_array_2.view(1, -1)
        dist_array_2 = dist_array_2[torch.gt(dist_array_2, 0)]
        top_k_2 = dist_array_2.sort()[0][-self.k:]
        print(top_k_2)
        '''
        dist_array = self._pairwise_distance(features)
        dist_array = dist_array.view(1, -1)
        top_k = dist_array.sort()[0][0, -self.k * 2::2]     # Because there are 2 same value of same feature pair in the dist_array
        # print('top k intra class dist:', top_k)
        return top_k 

def forward(self, tgt_seq, enc_output, enc_mask, enc_attn_caches=None, self_attn_caches=None):

        batch_size, tgt_len = tgt_seq.size()

        query_len = tgt_len
        key_len = tgt_len

        src_len = enc_output.size(1)

        # Run the forward pass of the TransformerDecoder.
        emb = self.embeddings(tgt_seq)

        if self_attn_caches is not None:
            emb = emb[:, -1:].contiguous()
            query_len = 1

        # Decode mask
        dec_slf_attn_pad_mask = tgt_seq.detach().eq(PAD).unsqueeze(1).expand(batch_size, query_len, key_len)
        dec_slf_attn_sub_mask = get_attn_causal_mask(emb)

        dec_slf_attn_mask = torch.gt(dec_slf_attn_pad_mask + dec_slf_attn_sub_mask, 0)
        dec_enc_attn_mask = enc_mask.unsqueeze(1).expand(batch_size, query_len, src_len)

        output = emb
        new_self_attn_caches = []
        new_enc_attn_caches = []
        for i in range(self.num_layers):
            output, attn, self_attn_cache, enc_attn_cache \
                = self.block_stack[i](output,
                                      enc_output,
                                      dec_slf_attn_mask,
                                      dec_enc_attn_mask,
                                      enc_attn_cache=enc_attn_caches[i] if enc_attn_caches is not None else None,
                                      self_attn_cache=self_attn_caches[i] if self_attn_caches is not None else None)

            new_self_attn_caches += [self_attn_cache]
            new_enc_attn_caches += [enc_attn_cache]

        output = self.out_layer_norm(output)

        return output, new_self_attn_caches, new_enc_attn_caches 

def forward(self, enc_outputs, enc_input, dec_input, dec_pos):
        dec_output = self.dec_ebd(dec_input) + self.pos_ebd(dec_pos)

        dec_slf_attn_mask = torch.gt(
            get_attn_padding_mask(dec_input, dec_input) + get_attn_subsequent_mask(dec_input), 0)
        dec_enc_attn_pad_mask = get_attn_padding_mask(dec_input, enc_input)

        for layer, enc_output in zip(self.decodes, enc_outputs):
            dec_output = layer(dec_output, enc_output,
                dec_slf_attn_mask, dec_enc_attn_pad_mask)

        return dec_output 

def forward(self, input, context, src_pad_mask, tgt_pad_mask):
        # Args Checks
        input_batch, input_len, _ = input.size()
        contxt_batch, contxt_len, _ = context.size()
        aeq(input_batch, contxt_batch)

        src_batch, t_len, s_len = src_pad_mask.size()
        tgt_batch, t_len_, t_len__ = tgt_pad_mask.size()
        aeq(input_batch, contxt_batch, src_batch, tgt_batch)
        aeq(t_len, t_len_, t_len__, input_len)
        aeq(s_len, contxt_len)
        # END Args Checks

        dec_mask = torch.gt(tgt_pad_mask + self.mask[:, :tgt_pad_mask.size(1),
                            :tgt_pad_mask.size(1)]
                            .expand_as(tgt_pad_mask), 0)
        input_norm = self.layer_norm_1(input)
        query, attn = self.self_attn(input_norm, input_norm, input_norm,
                                     mask=dec_mask)
        query_norm = self.layer_norm_2(query+input)
        mid, attn = self.context_attn(context, context, query_norm,
                                      mask=src_pad_mask)
        output = self.feed_forward(mid+query+input)

        # CHECKS
        output_batch, output_len, _ = output.size()
        aeq(input_len, output_len)
        aeq(contxt_batch, output_batch)

        n_batch_, t_len_, s_len_ = attn.size()
        aeq(input_batch, n_batch_)
        aeq(contxt_len, s_len_)
        aeq(input_len, t_len_)
        # END CHECKS

        return output, attn