Python chainer.functions.softmax_cross_entropy() Examples

def forward(net, image_batch, sentence_batch, train=True):
    images = xp.asarray(image_batch)
    n, sentence_length = sentence_batch.shape
    net.initialize(images)
    loss = 0
    acc = 0
    size = 0
    for i in range(sentence_length - 1):
        target = xp.where(xp.asarray(sentence_batch[:, i]) != eos, 1, 0).astype(np.float32)
        if (target == 0).all():
            break
        with chainer.using_config('train', train):
            with chainer.using_config('enable_backprop', train):
                x = xp.asarray(sentence_batch[:, i])
                t = xp.asarray(sentence_batch[:, i + 1])
                y = net(x)
                y_max_index = xp.argmax(y.data, axis=1)
                mask = target.reshape((len(target), 1)).repeat(y.data.shape[1], axis=1)
                y = y * mask
                loss += F.softmax_cross_entropy(y, t)
                acc += xp.sum((y_max_index == t) * target)
                size += xp.sum(target)
    return loss / size, float(acc) / size, float(size) 

def __call__(self, x, t):
        h = F.relu(self.conv1(x))
        h = F.max_pooling_2d(h, 2, 1)
        h = F.relu(self.conv2(h))
        h = F.relu(self.conv3(h))
        h = F.dropout(F.relu(self.fc4(h)), train=self.train)
        h = self.fc5(h)
        h = F.reshape(h, (x.data.shape[0], 3, 16, 16))
        h = self.channelwise_inhibited(h)

        if self.train:
            self.loss = F.softmax_cross_entropy(h, t, normalize=False)
            return self.loss
        else:
            self.pred = F.softmax(h)
            return self.pred 

def check_double_backward(self, xp):
        x = xp.asarray(self.x)
        t = xp.asarray(self.t)
        gy = xp.asarray(self.gy)
        ggx = xp.asarray(self.ggx)
        if self.class_weight is not None:
            class_weight = xp.asarray(self.class_weight)
        else:
            class_weight = None

        def f(x_):
            return functions.softmax_cross_entropy(
                x_, t, class_weight=class_weight, reduce=self.reduce,
                ignore_label=self.ignore_label,
                enable_double_backprop=True)

        gradient_check.check_double_backward(f, x, gy, ggx) 

def check_backward(self, xp):
        x = xp.asarray(self.x)
        t = xp.asarray(self.t)
        gy = xp.asarray(self.gy)
        if self.class_weight is not None:
            class_weight = xp.asarray(self.class_weight)
        else:
            class_weight = None

        def f(x_, t_):
            return functions.softmax_cross_entropy(
                x_, t_, class_weight=class_weight, reduce=self.reduce,
                ignore_label=self.ignore_label,
                enable_double_backprop=self.enable_double_backprop)

        gradient_check.check_backward(f, (x, t), gy) 

def forward(self, inputs, device):
        x, = inputs
        t = device.send(self.t)
        class_weight = device.send(self.class_weight)
        loss = functions.softmax_cross_entropy(
            x, t, normalize=self.normalize, reduce=self.reduce,
            cache_score=self.cache_score, class_weight=class_weight,
            enable_double_backprop=self.enable_double_backprop)

        if not (self.enable_double_backprop or device.xp is chainerx):
            assert (loss.creator.y is not None) == self.cache_score

        # All the loss values except those corresponding to the ignored label
        # must be positive.
        # TODO(niboshi): Use device.xp.where once chainerx supports it.
        assert numpy.where(
            backend.CpuDevice().send(t == -1),
            True,
            backend.CpuDevice().send(loss.array) > 0).all()

        return loss, 

def __call__(self, x, t):
        h = F.relu(self.conv1(x))
        h = F.max_pooling_2d(h, 2, 1)
        h = F.relu(self.conv2(h))
        h = F.relu(self.conv3(h))
        h = F.relu(self.fc4(h))
        h = self.fc5(h)
        h = F.reshape(h, (x.data.shape[0], 3, 16, 16))
        h = self.channelwise_inhibited(h)

        if self.train:
            self.loss = F.softmax_cross_entropy(h, t, normalize=False)
            return self.loss
        else:
            self.pred = F.softmax(h)
            return self.pred 

def forward_one_step(self, x_data, y_data, state, train=True, dropout_ratio=0.5):
        x = Variable(x_data, volatile=not train)
        t = Variable(y_data, volatile=not train)

        h0      = self.embed(x)
        h1_in   = self.l1_x(F.dropout(h0, ratio=dropout_ratio, train=train)) + self.l1_h(state['h1'])
        c1, h1  = F.lstm(state['c1'], h1_in)
        h2_in   = self.l2_x(F.dropout(h1, ratio=dropout_ratio, train=train)) + self.l2_h(state['h2'])
        c2, h2  = F.lstm(state['c2'], h2_in)
        y       = self.l3(F.dropout(h2, ratio=dropout_ratio, train=train))
        state   = {'c1': c1, 'h1': h1, 'c2': c2, 'h2': h2}

        if train:
            return state, F.softmax_cross_entropy(y, t)
        else:
            return state, F.softmax(y) 

def _get_loss_gen(self):
        batchsize = self.y_fake.data.shape[0]
        L_mce = F.softmax_cross_entropy(self.pred_label_map, self.ground_truth, normalize=False)
        L_bce = F.softmax_cross_entropy(self.y_fake, Variable(self.xp.ones(batchsize, dtype=self.xp.int32), volatile=not self.gen.train))
        loss = L_mce + self.L_bce_weight * L_bce

        # log report
        label_true = chainer.cuda.to_cpu(self.ground_truth.data)
        label_pred = chainer.cuda.to_cpu(self.pred_label_map.data).argmax(axis=1)
        logs = []
        for i in six.moves.range(batchsize):
            acc, acc_cls, iu, fwavacc = utils.label_accuracy_score(
                label_true[i], label_pred[i], self.n_class)
            logs.append((acc, acc_cls, iu, fwavacc))
        log = np.array(logs).mean(axis=0)
        values = {
            'loss': loss,
            'accuracy': log[0],
            'accuracy_cls': log[1],
            'iu': log[2],
            'fwavacc': log[3],
        }
        chainer.report(values, self.gen)

        return loss 

def calc_loss(self):
        batchsize = self.ground_truth.shape[0]
        self.loss = F.softmax_cross_entropy(self.pred_label_map, self.ground_truth, normalize=False)

        # log report
        label_true = chainer.cuda.to_cpu(self.ground_truth.data)
        label_pred = chainer.cuda.to_cpu(self.pred_label_map.data).argmax(axis=1)
        logs = []
        for i in six.moves.range(batchsize):
            acc, acc_cls, iu, fwavacc = utils.label_accuracy_score(
                label_true[i], label_pred[i], self.n_class)
            logs.append((acc, acc_cls, iu, fwavacc))
        log = np.array(logs).mean(axis=0)
        values = {
            'loss': self.loss,
            'accuracy': log[0],
            'accuracy_cls': log[1],
            'iu': log[2],
            'fwavacc': log[3],
        }
        chainer.report(values, self.model) 

def forward(self, x, t):
        # def forward(self, x):
        h = F.max_pooling_2d(F.local_response_normalization(
            F.relu(self.conv1(x))), 3, stride=2)
        h = F.max_pooling_2d(F.local_response_normalization(
            F.relu(self.conv2(h))), 3, stride=2)
        h = F.relu(self.conv3(h))
        h = F.relu(self.conv4(h))
        h = F.max_pooling_2d(F.relu(self.conv5(h)), 3, stride=2)
        h = F.dropout(F.relu(self.fc6(h)))
        h = F.dropout(F.relu(self.fc7(h)))
        h = self.fc8(h)

        loss = F.softmax_cross_entropy(h, t)
        #loss = h

        # chainer.report({'loss': loss, 'accuracy': F.accuracy(h, t)}, self)
        return loss

# from https://github.com/chainer/chainer/blob/master/examples/imagenet/alex.py 

def forward(self, img_feats, captions):
        """Batch of image features and image captions to a singe loss.

        Compute the softmax cross-entropy captioning loss in a single pass
        without iterating over the sequences.
        """
        hx, cx, _ = self.reset(img_feats)

        # Extract all inputs and targets for all captions in the batch
        xs = [c[:-1] for c in captions]  # del eos
        ts = [c[1:] for c in captions]  # del bos

        # Get the predictions `ys`
        _, _, ys = self.step(hx, cx, xs)

        # Since `ys` is concatenated, we also concatenate the target tokens
        # before computing the loss
        ts = F.concat(ts, axis=0)

        loss = F.softmax_cross_entropy(ys, ts)
        return loss 

def forward(self, x, t):
        # def forward(self, x):
        h = F.max_pooling_2d(F.local_response_normalization(
            F.relu(self.conv1(x))), 3, stride=2)
        h = F.max_pooling_2d(F.local_response_normalization(
            F.relu(self.conv2(h))), 3, stride=2)
        h = F.relu(self.conv3(h))
        h = F.relu(self.conv4(h))
        h = F.max_pooling_2d(F.relu(self.conv5(h)), 3, stride=2)
        h = F.dropout(F.relu(self.fc6(h)))
        h = F.dropout(F.relu(self.fc7(h)))
        h = self.fc8(h)

        loss = F.softmax_cross_entropy(h, t)
        #loss = h

        # chainer.report({'loss': loss, 'accuracy': F.accuracy(h, t)}, self)
        return loss 

def softmax_cross_entropy(self, y, t):
        import numpy as np

        log_softmax = F.log_softmax(y)
        # SelectItem is not supported by onnx-chainer.
        # TODO(hamaji): Support it?
        # log_prob = F.select_item(log_softmax, t)

        # TODO(hamaji): Currently, F.sum with axis=1 cannot be
        # backpropped properly.
        # log_prob = F.sum(log_softmax * t, axis=1)
        # self.batch_size = chainer.Variable(np.array(t.size, np.float32),
        #                                    name='batch_size')
        # return -F.sum(log_prob, axis=0) / self.batch_size
        log_prob = F.sum(log_softmax * t, axis=(0, 1))
        batch_size = chainer.Variable(np.array(t.shape[0], np.float32),
                                      name='batch_size')
        self.extra_inputs = [batch_size]
        loss = -log_prob / batch_size
        loss.name = 'loss'
        return loss 

def forward(self, x, t):
        h = self.bn1(self.conv1(x))
        h = F.max_pooling_2d(F.relu(h), 3, stride=2)
        h = self.res2(h)
        h = self.res3(h)
        h = self.res4(h)
        h = self.res5(h)
        h = F.average_pooling_2d(h, 7, stride=1)
        h = self.fc(h)

        #loss = F.softmax_cross_entropy(h, t)
        loss = self.softmax_cross_entropy(h, t)
        if self.compute_accuracy:
            chainer.report({'loss': loss, 'accuracy': F.accuracy(h, np.argmax(t, axis=1))}, self)
        else:
            chainer.report({'loss': loss}, self)
        return loss 

def softmax_cross_entropy(self, y, t):
        import numpy as np

        log_softmax = F.log_softmax(y)
        # SelectItem is not supported by onnx-chainer.
        # TODO(hamaji): Support it?
        # log_prob = F.select_item(log_softmax, t)

        # TODO(hamaji): Currently, F.sum with axis=1 cannot be
        # backpropped properly.
        # log_prob = F.sum(log_softmax * t, axis=1)
        # self.batch_size = chainer.Variable(np.array(t.size, np.float32),
        #                                    name='batch_size')
        # return -F.sum(log_prob, axis=0) / self.batch_size
        log_prob = F.sum(log_softmax * t, axis=(0, 1))
        batch_size = chainer.Variable(self.xp.array(t.shape[0], np.float32),
                                      name='batch_size')
        self.extra_inputs = [batch_size]
        loss = -log_prob / batch_size
        loss.name = 'loss'
        return loss 

def forward(self, x, t):
        h = F.max_pooling_2d(F.local_response_normalization(
            F.relu(self.conv1(x))), 3, stride=2)
        h = F.max_pooling_2d(F.local_response_normalization(
            F.relu(self.conv2(h))), 3, stride=2)
        h = F.relu(self.conv3(h))
        h = F.relu(self.conv4(h))
        h = F.max_pooling_2d(F.relu(self.conv5(h)), 3, stride=2)
        h = F.dropout(F.relu(self.fc6(h)))
        h = F.dropout(F.relu(self.fc7(h)))
        h = self.fc8(h)

        # EDIT(hamaji): ONNX-chainer cannot output SoftmaxCrossEntropy.
        # loss = F.softmax_cross_entropy(h, t)
        loss = self.softmax_cross_entropy(h, t)
        if self.compute_accuracy:
            chainer.report({'loss': loss, 'accuracy': F.accuracy(h, t)}, self)
        else:
            chainer.report({'loss': loss}, self)
        return loss 

def loss(self,es,x,y,t):
        """ Forward propagation and loss calculation
            Args:
                es (pair of ~chainer.Variable): encoder state 
                x (list of ~chainer.Variable): list of input sequences
                y (list of ~chainer.Variable): list of output sequences
                t (list of ~chainer.Variable): list of target sequences
                                   if t is None, it returns only states
            Return:
                es (pair of ~chainer.Variable(s)): encoder state
                ds (pair of ~chainer.Variable(s)): decoder state
                loss (~chainer.Variable) : cross-entropy loss
        """
        es,ey = self.encoder(es,x)
        ds,dy = self.decoder(es,y)
        if t is not None:
            loss = F.softmax_cross_entropy(dy,t)
            # avoid NaN gradients (See: https://github.com/pfnet/chainer/issues/2505)
            if chainer.config.train:
                loss += F.sum(F.concat(ey, axis=0)) * 0
            return es, ds, loss
        else: # if target is None, it only returns states
            return es, ds 

def loss(self,es,x,y,t):
        """ Forward propagation and loss calculation
            Args:
                es (pair of ~chainer.Variable): encoder state 
                x (list of ~chainer.Variable): list of input sequences
                y (list of ~chainer.Variable): list of output sequences
                t (list of ~chainer.Variable): list of target sequences
                                   if t is None, it returns only states
            Return:
                es (pair of ~chainer.Variable(s)): encoder state
                ds (pair of ~chainer.Variable(s)): decoder state
                loss (~chainer.Variable) : cross-entropy loss
        """
        es,ey = self.encoder(es,x)
        ds,dy = self.decoder(es,y)
        if t is not None:
            loss = F.softmax_cross_entropy(dy,t)
            # avoid NaN gradients (See: https://github.com/pfnet/chainer/issues/2505)
            if chainer.config.train:
                loss += F.sum(F.concat(ey, axis=0)) * 0
            return es, ds, loss
        else: # if target is None, it only returns states
            return es, ds 

def loss(self,es,x,y,t):
        """ Forward propagation and loss calculation
            Args:
                es (pair of ~chainer.Variable): encoder state 
                x (list of ~chainer.Variable): list of input sequences
                y (list of ~chainer.Variable): list of output sequences
                t (list of ~chainer.Variable): list of target sequences
                                   if t is None, it returns only states
            Return:
                es (pair of ~chainer.Variable(s)): encoder state
                ds (pair of ~chainer.Variable(s)): decoder state
                loss (~chainer.Variable) : cross-entropy loss
        """
        es,ey = self.encoder(es,x)
        ds,dy = self.decoder(es,y)
        if t is not None:
            loss = F.softmax_cross_entropy(dy,t)
            # avoid NaN gradients (See: https://github.com/pfnet/chainer/issues/2505)
            if chainer.config.train:
                loss += F.sum(F.concat(ey, axis=0)) * 0
            return es, ds, loss
        else: # if target is None, it only returns states
            return es, ds 

def loss(self,es,x,y,t):
        """ Forward propagation and loss calculation
            Args:
                es (pair of ~chainer.Variable): encoder state 
                x (list of ~chainer.Variable): list of input sequences
                y (list of ~chainer.Variable): list of output sequences
                t (list of ~chainer.Variable): list of target sequences
                                   if t is None, it returns only states
            Return:
                es (pair of ~chainer.Variable(s)): encoder state
                ds (pair of ~chainer.Variable(s)): decoder state
                loss (~chainer.Variable) : cross-entropy loss
        """
        es,ey = self.encoder(es,x)
        ds,dy = self.decoder(es,y)
        if t is not None:
            loss = F.softmax_cross_entropy(dy,t)
            # avoid NaN gradients (See: https://github.com/pfnet/chainer/issues/2505)
            if chainer.config.train:
                loss += F.sum(F.concat(ey, axis=0)) * 0
            return es, ds, loss
        else: # if target is None, it only returns states
            return es, ds 

def forward(self, x, t):
        h = self.bn1(self.conv1(x))
        h = F.max_pooling_2d(F.relu(h), 3, stride=2)
        h = self.res2(h)
        h = self.res3(h)
        h = self.res4(h)
        h = self.res5(h)
        h = F.average_pooling_2d(h, 7, stride=1)
        h = self.fc(h)

        loss = F.softmax_cross_entropy(h, t)
        chainer.report({'loss': loss, 'accuracy': F.accuracy(h, t)}, self)
        return loss 

def loss(
        self,
        words: [int],
        state: State,
        dropout: bool=True,
        train: bool=False
    ) -> Variable:
        if len(words) <= 1:
            raise Exception("word length error: >= 2")

        # convert words to variable
        _words = [
            Variable(self.xp.array([word], dtype=self.xp.int32))
            for word in words
        ]

        # predict next words
        ys, new_state = self.predictor(
            _words[:-1], state,
            dropout=dropout,
            train=train
        )

        # calculate loss
        loss = Variable(self.xp.zeros((), dtype=self.xp.float32))
        for y, t in zip(ys, _words[1:]):
            new_loss = F.softmax_cross_entropy(y, t)
            loss += new_loss

        len_words = Variable(self.xp.array(
            len(words) - 1,
            dtype=self.xp.float32
        ))
        return loss / len_words 

def loss_labeled(forward, x, t):
    y = forward(x, update_batch_stats=True)
    L = F.softmax_cross_entropy(y, t)
    return L 

def calc_loss(self, y, t):
        loss = F.softmax_cross_entropy(y, t)
        return loss 

def _train_linear_classifier(self, model, optimizer, backend_config):
        def _make_label(x):
            a = (numpy.dot(x, self.w) + self.b).reshape((self.BATCH_SIZE, ))
            t = numpy.empty_like(a).astype(numpy.int32)
            t[a >= 0] = 0
            t[a < 0] = 1
            return t

        def _make_dataset(batch_size, unit_num, dtype):
            x_data = numpy.random.uniform(
                -1, 1, (batch_size, unit_num)).astype(dtype)
            t_data = _make_label(x_data)
            x_data = backend_config.get_array(x_data)
            t_data = backend_config.get_array(t_data)
            x = chainer.Variable(x_data)
            t = chainer.Variable(t_data, requires_grad=False)
            return x, t

        for _ in six.moves.range(self.EPOCH):
            x, t = _make_dataset(self.BATCH_SIZE, self.UNIT_NUM, self.dtype)
            model.cleargrads()
            y = model(x)
            loss = F.softmax_cross_entropy(y, t)
            loss.backward()
            optimizer.update()

        x_test, t_test = _make_dataset(
            self.BATCH_SIZE, self.UNIT_NUM, self.dtype)
        y_test = model(x_test)
        return F.accuracy(y_test, t_test) 

def loss_test(forward, x, t):
    logit = forward(x, train=False)
    L, acc = F.softmax_cross_entropy(logit, t).data, F.accuracy(logit, t).data
    return L, acc 

def check_backward(self, xp):
        x = xp.asarray(self.x)
        t = xp.asarray(self.t)
        gy = None
        if self.reduce == 'no':
            gy = xp.asarray(self.gy)

        def f(x_, t_):
            return functions.softmax_cross_entropy(
                x_, t_, reduce=self.reduce)

        gradient_check.check_backward(f, (x, t), gy, dtype=numpy.float64,
                                      no_grads=(False, True),
                                      **self.check_backward_options) 

def check_consistency(self, xp):

        if self.class_weight is None:
            class_weight = None
        else:
            class_weight = xp.asarray(self.class_weight)

        x = xp.asarray(self.x)
        t = xp.asarray(self.t)

        def f(enable_double_backprop):
            kwargs = {
                'normalize': self.normalize,
                'class_weight': class_weight,
                'enable_double_backprop': enable_double_backprop
            }

            return functions.softmax_cross_entropy(x, t, **kwargs).data

        loss_single = f(False)
        loss_double = f(True)

        check_forward_options = {}
        if self.dtype == numpy.float16:
            check_forward_options = {'atol': 5e-4, 'rtol': 5e-3}
        testing.assert_allclose(
            loss_single, loss_double, **check_forward_options) 

def check_invalid_reduce(self, x, t):
        with self.assertRaises(ValueError):
            functions.softmax_cross_entropy(
                x, t,
                reduce='unknown_reduce_type',
                enable_double_backprop=self.enable_double_backprop) 

def check_forward(self, xp):
        x = xp.asarray(self.x)
        t = xp.asarray(self.t)
        loss = functions.softmax_cross_entropy(x, t, reduce=self.reduce)
        expect = functions.softmax_cross_entropy(
            x, xp.asarray(self.t_hard), reduce=self.reduce,
            soft_target_loss=self.soft_target_loss)
        testing.assert_allclose(loss.data, expect.data,
                                **self.check_forward_options)