Python torch.logsumexp() Examples

def forward(self, x, target):
        similarity_matrix = x @ x.T  # need gard here
        label_matrix = target.unsqueeze(1) == target.unsqueeze(0)
        negative_matrix = label_matrix.logical_not()
        positive_matrix = label_matrix.fill_diagonal_(False)

        sp = torch.where(positive_matrix, similarity_matrix,
                         torch.zeros_like(similarity_matrix))
        sn = torch.where(negative_matrix, similarity_matrix,
                         torch.zeros_like(similarity_matrix))

        ap = torch.clamp_min(1 + self.m - sp.detach(), min=0.)
        an = torch.clamp_min(sn.detach() + self.m, min=0.)

        logit_p = -self.gamma * ap * (sp - self.dp)
        logit_n = self.gamma * an * (sn - self.dn)

        logit_p = torch.where(positive_matrix, logit_p,
                              torch.zeros_like(logit_p))
        logit_n = torch.where(negative_matrix, logit_n,
                              torch.zeros_like(logit_n))

        loss = F.softplus(torch.logsumexp(logit_p, dim=1) +
                          torch.logsumexp(logit_n, dim=1)).mean()
        return loss 

def forward(self, tensor):
        tensor = self.blur(tensor)
        tensor = self.nonlinearity(tensor)

        centerbias = self.centerbias(tensor)
        if self.nonlinearity_target == 'density':
            tensor *= centerbias
        elif self.nonlineary_target == 'logdensity':
            tensor += centerbias
        else:
            raise ValueError(self.nonlinearity_target)

        if self.nonlinearity_target == 'density':
            sums = torch.sum(tensor, dim=(2, 3), keepdim=True)
            tensor = tensor / sums
            tensor = torch.log(tensor)
        elif self.nonlineary_target == 'logdensity':
            logsums = torch.logsumexp(tensor, dim=(2, 3), keepdim=True)
            tensor = tensor - logsums
        else:
            raise ValueError(self.nonlinearity_target)

        return tensor 

def forward(self, batch, labels):
        """
        Args:
            batch:   torch.Tensor() [(BS x embed_dim)], batch of embeddings
            labels:  np.ndarray [(BS x 1)], for each element of the batch assigns a class [0,...,C-1]
        Returns:
            proxynca loss (torch.Tensor(), batch-averaged)
        """
        #Normalize batch in case it is not normalized (which should never be the case for ProxyNCA, but still).
        #Same for the PROXIES. Note that the multiplication by 3 seems arbitrary, but helps the actual training.
        batch       = 3*torch.nn.functional.normalize(batch, dim=1)
        PROXIES     = 3*torch.nn.functional.normalize(self.PROXIES, dim=1)
        #Group required proxies
        pos_proxies = torch.stack([PROXIES[pos_label:pos_label+1,:] for pos_label in labels])
        neg_proxies = torch.stack([torch.cat([self.all_classes[:class_label],self.all_classes[class_label+1:]]) for class_label in labels])
        neg_proxies = torch.stack([PROXIES[neg_labels,:] for neg_labels in neg_proxies])
        #Compute Proxy-distances
        dist_to_neg_proxies = torch.sum((batch[:,None,:]-neg_proxies).pow(2),dim=-1)
        dist_to_pos_proxies = torch.sum((batch[:,None,:]-pos_proxies).pow(2),dim=-1)
        #Compute final proxy-based NCA loss
        negative_log_proxy_nca_loss = torch.mean(dist_to_pos_proxies[:,0] + torch.logsumexp(-dist_to_neg_proxies, dim=1))
        return negative_log_proxy_nca_loss 

def conditional_log_density(self, stimulus, x_hist, y_hist, t_hist, attributes=None, out=None):
        smap = self.parent_model.conditional_log_density(stimulus, x_hist, y_hist, t_hist, attributes=attributes, out=out)

        target_shape = (stimulus.shape[0],
                        stimulus.shape[1])

        if smap.shape != target_shape:
            if self.verbose:
                print("Resizing saliency map", smap.shape, target_shape)
            x_factor = target_shape[1] / smap.shape[1]
            y_factor = target_shape[0] / smap.shape[0]

            smap = zoom(smap, [y_factor, x_factor], order=1, mode='nearest')

            smap -= logsumexp(smap)

            assert smap.shape == target_shape

        return smap 

def forward(self, model, sample, reduce=True):
        """Compute the loss for the given sample.

        Returns a tuple with three elements:
        1) the loss, as a Variable
        2) the sample size, which is used as the denominator for the gradient
        3) logging outputs to display while training
        """
        translations, bleu_scores = self.generate_translations(model, sample)
        nll_loss = self.compute_nll(model, sample, translations)
        loss = nll_loss[:, 0] + torch.logsumexp(-nll_loss, 1)
        if reduce:
            loss = loss.sum()
        sample_size = (
            sample["target"].size(0) if self.args.sentence_avg else sample["ntokens"]
        )
        logging_output = {
            "loss": utils.item(loss.data) if reduce else loss.data,
            "ntokens": sample["ntokens"],
            "nsentences": sample["target"].size(0),
            "sample_size": sample_size,
        }
        return loss, sample_size, logging_output 

def forward(self, model, sample, reduce=True):
        """Compute the loss for the given sample.

        Returns a tuple with three elements:
        1) the loss, as a Variable
        2) the sample size, which is used as the denominator for the gradient
        3) logging outputs to display while training
        """
        translations, bleu_scores = self.generate_translations(model, sample)
        nll_loss = self.compute_nll(model, sample, translations)
        partition = torch.logsumexp(-nll_loss, 1)
        probs = torch.exp(-nll_loss - partition.unsqueeze(1))
        loss = torch.sum((1 - bleu_scores.cuda() / 100) * probs, dim=1)
        if reduce:
            loss = loss.sum()
        sample_size = (
            sample["target"].size(0) if self.args.sentence_avg else sample["ntokens"]
        )
        logging_output = {
            "loss": utils.item(loss.data) if reduce else loss.data,
            "ntokens": sample["ntokens"],
            "nsentences": sample["target"].size(0),
            "sample_size": sample_size,
        }
        return loss, sample_size, logging_output 

def _forward_alpha(self, emissions, M):
        Tt, B, Ts = emissions.size()
        alpha = utils.fill_with_neg_inf(torch.empty_like(emissions))  # Tt, B, Ts
        # initialization  t=1
        # initial = torch.empty_like(alpha[0]).fill_(-math.log(Ts))  # log(1/Ts)
        initial = utils.fill_with_neg_inf(torch.empty_like(alpha[0])) 
        initial[:, 0] = 0
        alpha[0] = emissions[0] + initial
        # print('Initialize alpha:', alpha[0])
        # induction
        for i in range(1, Tt):
            alpha[i] = torch.logsumexp(alpha[i-1].unsqueeze(-1) + M[i-1], dim=1)
            alpha[i] = alpha[i] + emissions[i]
            # print('Emissions@', i, emissions[i])
            # print('alpha@',i, alpha[i])
        return alpha 

def _log_density(self, stimulus):
        smap = self.parent_model.log_density(stimulus)

        target_shape = (stimulus.shape[0],
                        stimulus.shape[1])

        if smap.shape != target_shape:
            if self.verbose:
                print("Resizing saliency map", smap.shape, target_shape)
            x_factor = target_shape[1] / smap.shape[1]
            y_factor = target_shape[0] / smap.shape[0]

            smap = zoom(smap, [y_factor, x_factor], order=1, mode='nearest')

            smap -= logsumexp(smap)

            assert smap.shape == target_shape

        return smap 

def _prior_log_likelihood(self, z):
        """Computes the log likelihood of the prior, p(z), for different priors."""
        if self.prior == "weak":
            return self._sample_log_likelihood(z, torch.tensor([1.], device=self.device), 1)
        elif self.prior == "mog":
            mu, var = self._get_mixture_parameters()
            log_k = F.log_softmax(self.mixture_weights)
            # [batch_size, num_mixture]
            mixture_log = self._sample_log_likelihood(z.unsqueeze(
                1), torch.tensor([[1.]], device=self.device), dim=2, mu=mu, var=var)
            # [batch_size]
            return torch.logsumexp(mixture_log + log_k, dim=1)
        elif self.prior == "vamp":
            # [num_pseudo, z_dim]
            mu, var = self.encoder(self.pseudo_inputs, self.pseudo_lengths)
            # [num_pseudo, ]
            log_k = F.log_softmax(self.pseudo_weights)

            # [batch_size, num_pseudo]
            pseudo_log = self._sample_log_likelihood(z.unsqueeze(1), torch.tensor(
                [[1.]], device=self.device), dim=2, mu=mu, var=var)
            # [batch_size, ]
            return torch.logsumexp(pseudo_log + log_k, dim=1) 

def _computes_transition(self, prev_log_prob, path, path_lens, cum_log_prob, y, skip_accum=False):
        bs, max_path_len = path.size()
        mat = prev_log_prob.new_zeros(3, bs, max_path_len).fill_(self.log0)
        mat[0, :, :] = prev_log_prob
        mat[1, :, 1:] = prev_log_prob[:, :-1]
        mat[2, :, 2:] = prev_log_prob[:, :-2]
        # disable transition between the same symbols
        # (including blank-to-blank)
        same_transition = (path[:, :-2] == path[:, 2:])
        mat[2, :, 2:][same_transition] = self.log0
        log_prob = torch.logsumexp(mat, dim=0)
        outside = torch.arange(max_path_len, dtype=torch.int64) >= path_lens.unsqueeze(1)
        log_prob[outside] = self.log0
        if not skip_accum:
            cum_log_prob += log_prob
        batch_index = torch.arange(bs, dtype=torch.int64).unsqueeze(1)
        log_prob += y[batch_index, path]
        return log_prob 

def test_logsumexp():
    inputs = torch.tensor([
        0.5, 0.5, 0.0, -2.1, 3.2, 7.0, -1.0, -100.0,
        float('-inf'),
        float('-inf'), 0.0
    ])
    inputs.requires_grad_()
    index = torch.tensor([0, 0, 1, 1, 1, 2, 4, 4, 5, 6, 6])
    splits = [2, 3, 1, 0, 2, 1, 2]

    outputs = scatter_logsumexp(inputs, index)

    for src, out in zip(inputs.split(splits), outputs.unbind()):
        assert out.tolist() == torch.logsumexp(src, dim=0).tolist()

    outputs.backward(torch.randn_like(outputs)) 

def _forward_alpha(self, emissions, M):
        Tt, B, Ts = emissions.size()
        alpha = utils.fill_with_neg_inf(torch.empty_like(emissions))  # Tt, B, Ts
        # initialization  t=1
        initial = torch.empty_like(alpha[0]).fill_(-math.log(Ts))  # log(1/Ts)
        # initial = utils.fill_with_neg_inf(torch.empty_like(alpha[0])) 
        # initial[:, 0] = 0
        alpha[0] = emissions[0] + initial
        # print('Initialize alpha:', alpha[0])
        # induction
        for i in range(1, Tt):
            alpha[i] = torch.logsumexp(alpha[i-1].unsqueeze(-1) + M[i-1], dim=1)
            alpha[i] = alpha[i] + emissions[i]
            # print('Emissions@', i, emissions[i])
            # print('alpha@',i, alpha[i])
        return alpha 

def semantic_loss_exactly_one(log_prob):
    """Semantic loss to encourage the multinomial probability to be "peaked",
    i.e. only one class is picked.
    The loss has the form -log sum_{i=1}^n p_i prod_{j=1, j!=i}^n (1 - p_j).
    Paper: http://web.cs.ucla.edu/~guyvdb/papers/XuICML18.pdf
    Code: https://github.com/UCLA-StarAI/Semantic-Loss/blob/master/semi_supervised/semantic.py
    Parameters:
        log_prob: log probability of a multinomial distribution, shape (batch_size, n)
    Returns:
        semantic_loss: shape (batch_size)
    """
    _, argmaxes = torch.max(log_prob, dim=-1)
    # Compute log(1-p) separately for the largest probabilities, by doing
    # logsumexp on the rest of the log probabilities.
    log_prob_temp = log_prob.clone()
    log_prob_temp[range(log_prob.shape[0]), argmaxes] = torch.tensor(float('-inf'))
    log_1mprob_max = torch.logsumexp(log_prob_temp, dim=-1)
    # Compute log(1-p) normally for the rest of the probabilities
    log_1mprob = torch.log1p(-torch.exp(log_prob_temp))
    log_1mprob[range(log_prob.shape[0]), argmaxes] = log_1mprob_max
    loss = -(log_1mprob.sum(dim=-1) + torch.logsumexp(log_prob - log_1mprob, dim=-1))
    return loss 

def test_semantic_loss_exactly_one():
    m = 5
    logit = torch.randn(m)
    p = nn.functional.softmax(logit, dim=-1)
    # Compute manually
    result = 0.0
    for i in range(m):
        prod = p[i].clone()
        for j in range(m):
            if j != i:
                prod *= 1 - p[j]
        result += prod
    result = -torch.log(result)
    result1 = -torch.logsumexp(torch.log(1 - p).sum() + torch.log(p / (1 - p)), dim=-1)
    result2 = semantic_loss_exactly_one(p.unsqueeze(0)).squeeze()
    assert torch.allclose(result, result1)
    assert torch.allclose(result, result2) 

def compute_diag_log_prob(preds_mean, preds_log_cov, true_outputs, n_samples):
    '''
        Compute log prob assuming diagonal Gaussian with some mean and log cov
    '''
    preds_cov = torch.exp(preds_log_cov)

    log_prob = -0.5 * torch.sum(
        (preds_mean - repeated_true_outputs)**2 / preds_cov
    )

    log_det = torch.logsumexp(torch.sum(preds_log_cov, 1))
    log_det += log_2pi
    log_det *= -0.5
    log_prob += log_det

    log_prob /= float(n_samples)
    return log_prob 

def _forward_alpha(self, emissions, M):
        Tt, B, Ts = emissions.size()
        alpha = utils.fill_with_neg_inf(torch.empty_like(emissions))  # Tt, B, Ts
        # initialization  t=1
        initial = torch.empty_like(alpha[0]).fill_(-math.log(Ts))  # log(1/Ts)
        # initial = utils.fill_with_neg_inf(torch.empty_like(alpha[0])) 
        # initial[:, 0] = 0
        alpha[0] = emissions[0] + initial
        # print('Initialize alpha:', alpha[0])
        # induction
        for i in range(1, Tt):
            alpha[i] = torch.logsumexp(alpha[i-1].unsqueeze(-1) + M[i-1], dim=1)
            alpha[i] = alpha[i] + emissions[i]
            # print('Emissions@', i, emissions[i])
            # print('alpha@',i, alpha[i])
        return alpha 

def forward(self, scores, sample_density, gt_density, mc_dim=-1):
        """Args:
            scores: predicted score values
            sample_density: probability density of the sample distribution
            gt_density: probability density of the ground truth distribution
            mc_dim: dimension of the MC samples"""

        exp_val = scores - torch.log(sample_density + self.eps)

        L = torch.logsumexp(exp_val, dim=mc_dim) - math.log(scores.shape[mc_dim]) - \
            torch.mean(scores * (gt_density / (sample_density + self.eps)), dim=mc_dim)

        return L.mean() 

def _backward_beta(self, emissions, M):
        Tt, B, Ts = emissions.size()
        beta = utils.fill_with_neg_inf(torch.empty_like(emissions))  # Tt, B, Ts
        # initialization
        beta[-1] = 0
        for i in range(Tt-2, -1, -1):
            beta[i] = torch.logsumexp(M[i].transpose(1, 2) +  # N, Ts, Ts
                                      beta[i+1].unsqueeze(-1) +  # N, Ts, 1
                                      emissions[i+1].unsqueeze(-1),  # N, Ts, 1
                                      dim=1)
        return beta 

def forward(self, scores, sample_density, gt_density=None, mc_dim=-1):
        """Args:
            scores: predicted score values. First sample must be ground-truth
            sample_density: probability density of the sample distribution
            gt_density: not used
            mc_dim: dimension of the MC samples. Only mc_dim=1 supported"""

        assert mc_dim == 1
        assert (sample_density[:,0,...] == -1).all()

        exp_val = scores[:, 1:, ...] - torch.log(sample_density[:, 1:, ...] + self.eps)

        L = torch.logsumexp(exp_val, dim=mc_dim) - math.log(scores.shape[mc_dim] - 1) - scores[:, 0, ...]
        loss = L.mean()
        return loss 

def log_prob(self, value, sum=False):
        if self._batch_length == 0:
            value = util.to_tensor(value).squeeze()
            lp = torch.logsumexp(self._log_probs + util.to_tensor([d.log_prob(value) for d in self._distributions]), dim=0)
        else:
            value = util.to_tensor(value).view(self._batch_length)
            lp = torch.logsumexp(self._log_probs + torch.stack([d.log_prob(value).squeeze(-1) for d in self._distributions]).view(-1, self._batch_length).t(), dim=1)
        return torch.sum(lp) if sum else lp 

def aggregate_accuracy(test_logits_sample, test_labels):
    """
    Compute classification accuracy.
    """
    averaged_predictions = torch.logsumexp(test_logits_sample, dim=0)
    return torch.mean(torch.eq(test_labels, torch.argmax(averaged_predictions, dim=-1)).float()) 

def _backward_beta(self, emissions, M):
        Tt, B, Ts = emissions.size()
        beta = utils.fill_with_neg_inf(torch.empty_like(emissions))  # Tt, B, Ts
        # initialization
        beta[-1] = 0
        for i in range(Tt-2, -1, -1):
            beta[i] = torch.logsumexp(M[i].transpose(1, 2) +  # N, Ts, Ts
                                      beta[i+1].unsqueeze(-1) +  # N, Ts, 1
                                      emissions[i+1].unsqueeze(-1),  # N, Ts, 1
                                      dim=1)
        return beta 

def forward(self, pred, target, C):
		u = torch.zeros_like(pred)
		v = torch.zeros_like(target)

		actual_nits = 0 # To check if algorithm terminates because of threshold or max iterations reached
		thresh = 1e-1   # Stopping criterion

		## Sinkhorn iterations ##
		for i in range(self.max_iter):
			u1 = u  # useful to check the update
			u = self.eps * (torch.log(pred + 1e-8) - torch.logsumexp(self.M(C, u, v), -1)) + u
			v = self.eps * (torch.log(target + 1e-8) - torch.logsumexp(self.M(C, u, v).transpose(-2, -1), -1)) + v
			err = (u - u1).abs().sum(-1).mean()

			actual_nits += 1
			if err.item() < thresh:
				break

		U, V = u, v
		pi = torch.exp(self.M(C, U, V))         # Transport plan pi = diag(a)*K*diag(b)
		dist = torch.sum(pi * C, dim=(-2, -1))  # Sinkhorn distance

		"""
		if self.reduction == 'mean':
			dist = dist.mean()
		elif self.reduction == 'sum':
			dist = dist.sum()
		"""

		return dist, pi 

def log_sum_exp(x, axis = 1):
    m = torch.max(x, keepdim = True)
    return m + torch.logsumexp(x - m, dim = 1, keepdim = True) 

def __call__(self, _, global_features, targets):
        global_features = F.normalize(global_features, dim=1)

        sim_mat = torch.matmul(global_features, global_features.t())

        N = sim_mat.size(0)
        is_pos = targets.expand(N, N).eq(targets.expand(N, N).t()).float() - torch.eye(N).to(sim_mat.device)
        is_pos = is_pos.bool()
        is_neg = targets.expand(N, N).ne(targets.expand(N, N).t())

        s_p = sim_mat[is_pos].contiguous().view(N, -1)
        s_n = sim_mat[is_neg].contiguous().view(N, -1)

        alpha_p = F.relu(-s_p.detach() + 1 + self.m)
        alpha_n = F.relu(s_n.detach() + self.m)
        delta_p = 1 - self.m
        delta_n = self.m

        logit_p = - self.s * alpha_p * (s_p - delta_p)
        logit_n = self.s * alpha_n * (s_n - delta_n)

        loss = F.softplus(torch.logsumexp(logit_p, dim=1) + torch.logsumexp(logit_n, dim=1)).mean()

        return {
            "loss_circle": loss * self._scale,
        } 

def forward(self, inputs):
		'''
		Forward pass, return log probabilities over correspondences.

		inputs -- 4D data tensor (BxCxNx1)
		B -> batch size (multiple image pairs)
		C -> 5 values (2D coordinate + 2D coordinate + 1D side information)
		N -> number of correspondences
		1 -> dummy dimension
		
		'''
		batch_size = inputs.size(0)
		data_size = inputs.size(2) # number of correspondences

		x = inputs
		x = F.relu(self.p_in(x))
		
		for r in self.res_blocks:
			res = x
			x = F.relu(r[1](F.instance_norm(r[0](x)))) 
			x = F.relu(r[3](F.instance_norm(r[2](x))))
			x = x + res

		log_probs = F.logsigmoid(self.p_out(x))

		# normalization in log space such that probabilities sum to 1
		log_probs = log_probs.view(batch_size, -1)
		normalizer = torch.logsumexp(log_probs, dim=1)
		normalizer = normalizer.unsqueeze(1).expand(-1, data_size)
		log_probs = log_probs - normalizer
		log_probs = log_probs.view(batch_size, 1, data_size, 1)

		return log_probs 

def forward(self, observations, emissions):
        """
        Inputs: 
            observations : Input for the controller, B, Tt, Ts, C
            emissions: Output emissions, B, Tt, Ts

        """
        controls = self.predict_read_write(observations).permute(1,0,2,3)  # Tt,B,Ts,2
        Tt, B, Ts = emissions.size()
        # E-step
        with torch.no_grad():
            # get transition matrix:
            M = self.get_transitions(controls.clone())  # Tt, B, Ts, Ts
            alpha = self._forward_alpha(emissions, M)
            beta = self._backward_beta(emissions, M)
            prior = torch.logsumexp(alpha[-1:], dim=-1, keepdim=True)

            # Sanity check:
            # prior_1 = torch.sum(torch.exp(alpha[1]) * torch.exp(beta[1]), dim=-1)
            # prior_2 = torch.sum(torch.exp(alpha[2]) * torch.exp(beta[2]), dim=-1)
            # print('Prior with n=1:', prior_1, 'Prior with n=2', prior_2, 'Prior with n=-1:', torch.exp(prior.squeeze(-1)))
            
            # Posteriors:
            gamma = alpha + beta - prior
            gamma = torch.exp(gamma)  # Tt, N, Ts
            ksi = alpha[:-1].unsqueeze(-1) + beta[1:].unsqueeze(-2) + emissions[1:].unsqueeze(-2) + M[:-1] - prior.unsqueeze(-1)
            ksi = torch.exp(ksi)

            # Get read/write labels from posteriors:
            write = gamma[1:]
            read = torch.ones_like(write)
            for t in range(1, Tt):
                for j in range(Ts):
                    read[t-1, :, j] = ksi[t-1, :, :j+1, j+1:].sum(dim=-1).sum(dim=-1)
        return controls[:-1], gamma, read, write 

def logsumexp(a, b):
    m = torch.max(a, b)
    return torch.log(torch.exp(a - m) + torch.exp(b - m)) 

def logsumexp(a, b):
    m = torch.max(a, b)
    return torch.log(torch.exp(a - m) + torch.exp(b - m)) 

def effective_sample_size(self):
        self._check_finalized()
        if self._effective_sample_size is None:
            weights = self._categorical.probs
            self._effective_sample_size = 1. / weights.pow(2).sum()
            # log_weights = self._categorical.logits
            # self._effective_sample_size = torch.exp(2. * torch.logsumexp(log_weights, dim=0) - torch.logsumexp(2. * log_weights, dim=0))
        return self._effective_sample_size