Python chainer.functions.dropout() Examples

def __call__(self, x, mask):
        self.m.W.data = self.xp.array(self.maskW) #mask windows are set by 1
        h = self.c(x*mask) #(B,C,H,W)
        B,C,H,W = h.shape
        b = F.transpose(F.broadcast_to(self.c.b,(B,H,W,C)),(0,3,1,2))
        h = h - b
        mask_sums = self.m(mask)
        mask_new = (self.xp.sign(mask_sums.data-0.5)+1.0)*0.5
        mask_new_b = mask_new.astype("bool")
        
        mask_sums = F.where(mask_new_b,mask_sums,0.01*Variable(self.xp.ones(mask_sums.shape).astype("f")))
        h = h/mask_sums + b
         
        mask_new = Variable(mask_new)
        h = F.where(mask_new_b, h, Variable(self.xp.zeros(h.shape).astype("f"))) 

        if self.bn:
            h = self.batchnorm(h)
        if self.noise:
            h = add_noise(h)
        if self.dropout:
            h = F.dropout(h)
        if not self.activation is None:
            h = self.activation(h)
        return h, mask_new 

def __call__(self, x):
        h = F.relu(self.conv1_1(x))
        h = F.relu(self.conv1_2(h))
        h = F.max_pooling_2d(h, 2, stride=2)
        h = F.relu(self.conv2_1(h))
        h = F.relu(self.conv2_2(h))
        h = F.max_pooling_2d(h, 2, stride=2)
        h = F.relu(self.conv3_1(h))
        h = F.relu(self.conv3_2(h))
        h = F.max_pooling_2d(h, 2, stride=2)
        h = F.relu(self.conv4_1(h))
        h = F.relu(self.conv4_2(h))
        h = F.spatial_pyramid_pooling_2d(h, 3, F.MaxPooling2D)
        h = F.tanh(self.fc4(h))
        h = F.dropout(h, ratio=.5, train=self.train)
        h = F.tanh(self.fc5(h))
        h = F.dropout(h, ratio=.5, train=self.train)
        h = self.fc6(h)
        return h 

def __call__(self, x):
        if self.dr:
            x = F.dropout(x, self.dr)
        x = F.transpose(x, (0, 2, 1, 3))
        out_shape = list(x.shape)
        x = F.reshape(x, (-1, x.shape[2]*x.shape[3]))
        x = self.l(x)
        x = self.activation(x)
        out_shape[2] = self.out_ch
        x = F.reshape(x, out_shape)
        x = F.transpose(x, (0, 2, 1, 3))
        return x 

def __init__(self, V, d_model=512, n_heads=8, d_ff=2048, experimental_relu=False, dropout=None, nb_layers=6,
                 residual_mode="normal", no_normalize=False):
        super(Decoder, self).__init__(
            emb = L.EmbedID(V, d_model),
            encoding_layers = DecoderMultiLayer(d_model, n_heads, d_ff=d_ff,
                                                experimental_relu=experimental_relu, 
                                                dropout=dropout, nb_layers=nb_layers,
                                                residual_mode=residual_mode, no_normalize=no_normalize),
            logits_layer = L.Linear(d_model, V + 1)
        )
        
        self.dropout = dropout
        self.n_heads = n_heads
        self.d_model = d_model
        self.cached_pos_vect = None
        
        self.add_param("bos_encoding", (1, 1, d_model))
        self.bos_encoding.data[...] = np.random.randn(d_model)
        
        self.V = V
        self.eos_idx = V 

def __call__(self, x):
        # x: (B, T, F) ... batch, time, (mel)freq
        BT_size = x.shape[0] * x.shape[1]
        # e: (BT, F)
        e = self.linear_in(x.reshape(BT_size, -1))
        # Encoder stack
        for i in range(self.n_layers):
            # layer normalization
            e = getattr(self, '{}{:d}'.format("lnorm1_", i))(e)
            # self-attention
            s = getattr(self, '{}{:d}'.format("self_att_", i))(e, x.shape[0])
            # residual
            e = e + F.dropout(s, self.dropout)
            # layer normalization
            e = getattr(self, '{}{:d}'.format("lnorm2_", i))(e)
            # positionwise feed-forward
            s = getattr(self, '{}{:d}'.format("ff_", i))(e)
            # residual
            e = e + F.dropout(s, self.dropout)
        # final layer normalization
        # output: (BT, F)
        return self.lnorm_out(e) 

def __call__(self, x):
        h = F.relu(self.conv1_1(x))
        h = F.relu(self.conv1_2(h))
        h = F.max_pooling_2d(h, 2, stride=2)
        h = F.relu(self.conv2_1(h))
        h = F.relu(self.conv2_2(h))
        h = F.max_pooling_2d(h, 2, stride=2)
        h = F.relu(self.conv3_1(h))
        h = F.relu(self.conv3_2(h))
        h = F.relu(self.conv3_3(h))
        h = F.max_pooling_2d(h, 2, stride=2)
        h = F.relu(self.conv4_1(h))
        h = F.relu(self.conv4_2(h))
        h = F.relu(self.conv4_3(h))
        h = F.max_pooling_2d(h, 2, stride=2)
        h = F.relu(self.conv5_1(h))
        h = F.relu(self.conv5_2(h))
        h = F.relu(self.conv5_3(h))
        h = F.max_pooling_2d(h, 2, stride=2)
        h = F.dropout(F.relu(self.fc6(h)), train=self.train, ratio=0.5)
        h = F.dropout(F.relu(self.fc7(h)), train=self.train, ratio=0.5)
        h = self.fc8(h)
        return h 

def residual(self, x):
        h = x
        h = self.c1(h)
        if self.bn:
            h = self.b1(h)
        if self.activation:
            h = self.activation(h)
        if self.mode:
            h = self.mode(h)
        if self.dr:
            with chainer.using_config('train', True):
                h = F.dropout(h, self.dr)
        h = self.c2(h)
        if self.bn:
            h = self.b2(h)
        if self.activation:
            h = self.activation(h)
        return h 

def forward_one_step(self, x_data, y_data, state, train=True, dropout_ratio=0.5):
        x = Variable(x_data, volatile=not train)
        t = Variable(y_data, volatile=not train)

        h0      = self.embed(x)
        h1_in   = self.l1_x(F.dropout(h0, ratio=dropout_ratio, train=train)) + self.l1_h(state['h1'])
        c1, h1  = F.lstm(state['c1'], h1_in)
        h2_in   = self.l2_x(F.dropout(h1, ratio=dropout_ratio, train=train)) + self.l2_h(state['h2'])
        c2, h2  = F.lstm(state['c2'], h2_in)
        y       = self.l3(F.dropout(h2, ratio=dropout_ratio, train=train))
        state   = {'c1': c1, 'h1': h1, 'c2': c2, 'h2': h2}

        if train:
            return state, F.softmax_cross_entropy(y, t)
        else:
            return state, F.softmax(y) 

def __init__(self, idim, n_layers, n_units,
                 e_units=2048, h=8, dropout=0.1):
        super(TransformerEncoder, self).__init__()
        with self.init_scope():
            self.linear_in = L.Linear(idim, n_units)
            self.lnorm_in = L.LayerNormalization(n_units)
            self.pos_enc = PositionalEncoding(n_units, dropout, 5000)
            self.n_layers = n_layers
            self.dropout = dropout
            for i in range(n_layers):
                setattr(self, '{}{:d}'.format("lnorm1_", i),
                        L.LayerNormalization(n_units))
                setattr(self, '{}{:d}'.format("self_att_", i),
                        MultiHeadSelfAttention(n_units, h))
                setattr(self, '{}{:d}'.format("lnorm2_", i),
                        L.LayerNormalization(n_units))
                setattr(self, '{}{:d}'.format("ff_", i),
                        PositionwiseFeedForward(n_units, e_units, dropout))
            self.lnorm_out = L.LayerNormalization(n_units) 

def block_embed(embed, x, dropout=0.):
    """Embedding function followed by convolution

    Args:
        embed (callable): A :func:`~chainer.functions.embed_id` function
            or :class:`~chainer.links.EmbedID` link.
        x (:class:`~chainer.Variable` or :class:`numpy.ndarray` or \
        :class:`cupy.ndarray`): Input variable, which
            is a :math:`(B, L)`-shaped int array. Its first dimension
            :math:`(B)` is assumed to be the *minibatch dimension*.
            The second dimension :math:`(L)` is the length of padded
            sentences.
        dropout (float): Dropout ratio.

    Returns:
        ~chainer.Variable: Output variable. A float array with shape
        of :math:`(B, N, L, 1)`. :math:`(N)` is the number of dimensions
        of word embedding.

    """
    e = embed(x)
    e = F.dropout(e, ratio=dropout)
    e = F.transpose(e, (0, 2, 1))
    e = e[:, :, :, None]
    return e 

def residual(self, x):
        h = x
        h = self.c1(h)
        if self.bn:
            h = self.b1(h)
        if self.activation:
            h = self.activation(h)
        if self.mode:
            h = self.mode(h)
        if self.dr:
            with chainer.using_config('train', True):
                h = F.dropout(h, self.dr)
        h = self.c2(h)
        if self.bn:
            h = self.b2(h)
        if self.activation:
            h = self.activation(h)
        return h 

def __init__(self, n_layers, n_vocab, embed_size, hidden_size, dropout=0.1):
        hidden_size /= 3
        super(CNNEncoder, self).__init__(
            embed=L.EmbedID(n_vocab, embed_size, ignore_label=-1,
                            initialW=embed_init),
            cnn_w3=L.Convolution2D(
                embed_size, hidden_size, ksize=(3, 1), stride=1, pad=(2, 0),
                nobias=True),
            cnn_w4=L.Convolution2D(
                embed_size, hidden_size, ksize=(4, 1), stride=1, pad=(3, 0),
                nobias=True),
            cnn_w5=L.Convolution2D(
                embed_size, hidden_size, ksize=(5, 1), stride=1, pad=(4, 0),
                nobias=True),
            mlp=MLP(n_layers, hidden_size * 3, dropout)
        )
        self.output_size = hidden_size * 3
        self.dropout = dropout 

def __init__(self, d_model, n_heads, d_ff=2048, experimental_relu=False, dropout=None,
                 residual_mode="normal", no_normalize=False):
        super(DecoderLayer, self).__init__(
            ff_layer = FeedForward(d_model, d_ff=d_ff, dropout=dropout, residual_mode=residual_mode, no_normalize=no_normalize),
            self_attention_layer = AddAndNormalizedSelfAttentionLayer(d_model=d_model, n_heads=n_heads,
                                                             experimental_relu=experimental_relu,
                                                          dropout=dropout, residual_mode=residual_mode, no_normalize=no_normalize),
            
            cross_attention_layer = AddAndNormalizedCrossAttentionLayer(d_model=d_model, n_heads=n_heads,
                                                             experimental_relu=experimental_relu,
                                                          dropout=dropout, 
                                        residual_mode=residual_mode if residual_mode != "none" else "normal", no_normalize=no_normalize) # Does not seem good to not let the cross attention be bypassed
        )
        
        self.n_heads = n_heads
        self.d_model = d_model 

def __call__(self, sub_output, inpt):
        if self.dropout is not None:
            sub_output = F.dropout(sub_output, ratio=self.dropout)
            
        if self.residual_mode == "normal":
            added_output = sub_output + inpt
        else:
            added_output = sub_output
        
        if self.no_normalize:
            final_layer = added_output
        else:
            final_layer = self.apply_layer_normalization(added_output)

        if self.residual_mode == "after":
            final_layer = final_layer + inpt

        return final_layer


########################################################################
# Feed Forward layer with pass-through and normalization
# 

def forward(self, x, t):
        # def forward(self, x):
        h = F.max_pooling_2d(F.local_response_normalization(
            F.relu(self.conv1(x))), 3, stride=2)
        h = F.max_pooling_2d(F.local_response_normalization(
            F.relu(self.conv2(h))), 3, stride=2)
        h = F.relu(self.conv3(h))
        h = F.relu(self.conv4(h))
        h = F.max_pooling_2d(F.relu(self.conv5(h)), 3, stride=2)
        h = F.dropout(F.relu(self.fc6(h)))
        h = F.dropout(F.relu(self.fc7(h)))
        h = self.fc8(h)

        loss = F.softmax_cross_entropy(h, t)
        #loss = h

        # chainer.report({'loss': loss, 'accuracy': F.accuracy(h, t)}, self)
        return loss

# from https://github.com/chainer/chainer/blob/master/examples/imagenet/alex.py 

def __init__(self, n_layers, in_size, out_size, embed_size, hidden_size, proj_size, dropout=0.5):
        """Initialize encoder with structure parameters

        Args:
            n_layers (int): Number of layers.
            in_size (int): Dimensionality of input vectors.
            out_size (int): Dimensionality of output vectors.
            embed_size (int): Dimensionality of word embedding.
            hidden_size (int) : Dimensionality of hidden vectors.
            proj_size (int) : Dimensionality of projection before softmax.
            dropout (float): Dropout ratio.
        """
        super(LSTMDecoder, self).__init__(
            embed = L.EmbedID(in_size, embed_size),
            lstm = L.NStepLSTM(n_layers, embed_size, hidden_size, dropout),
            proj = L.Linear(hidden_size, proj_size),
            out = L.Linear(proj_size, out_size)
        )
        self.dropout = dropout
        for param in self.params():
            param.data[...] = np.random.uniform(-0.1, 0.1, param.data.shape) 

def __init__(self, n_layers, in_size, out_size, embed_size, hidden_size, proj_size, dropout=0.5):
        """Initialize encoder with structure parameters

        Args:
            n_layers (int): Number of layers.
            in_size (int): Dimensionality of input vectors.
            out_size (int): Dimensionality of output vectors.
            embed_size (int): Dimensionality of word embedding.
            hidden_size (int) : Dimensionality of hidden vectors.
            proj_size (int) : Dimensionality of projection before softmax.
            dropout (float): Dropout ratio.
        """
        super(LSTMDecoder, self).__init__(
            embed = L.EmbedID(in_size, embed_size),
            lstm = L.NStepLSTM(n_layers, embed_size, hidden_size, dropout),
            proj = L.Linear(hidden_size, proj_size),
            out = L.Linear(proj_size, out_size)
        )
        self.dropout = dropout
        for param in self.params():
            param.data[...] = np.random.uniform(-0.1, 0.1, param.data.shape) 

def __call__(self, x, t):
        h = F.relu(self.conv1_1(x))
        h = F.relu(self.conv1_2(h))
        h = F.max_pooling_2d(h, 2, 2)
        h = F.relu(self.conv2_1(h))
        h = F.relu(self.conv2_2(h))
        h = F.max_pooling_2d(h, 2, 2)
        h = F.relu(self.conv3_1(h))
        h = F.relu(self.conv3_2(h))
        h = F.relu(self.conv3_3(h))
        h = F.max_pooling_2d(h, 2, 2)
        h = F.relu(self.conv4_1(h))
        h = F.relu(self.conv4_2(h))
        h = F.relu(self.conv4_3(h))
        h = F.max_pooling_2d(h, 2, 2)
        h = F.relu(self.conv5_1(h))
        h = F.relu(self.conv5_2(h))
        h = F.relu(self.conv5_3(h))
        h = F.max_pooling_2d(h, 2, 2)
        h = F.dropout(F.relu(self.fc6(h)), ratio=0.5, train=self.train)
        h = F.dropout(F.relu(self.fc7(h)), ratio=0.5, train=self.train)
        h = self.score_fr(h)
        h = self.upsample(h)

        return h 

def __init__(self, n_layers, in_size, out_size, embed_size, hidden_size, proj_size, dropout=0.5):
        """Initialize encoder with structure parameters

        Args:
            n_layers (int): Number of layers.
            in_size (int): Dimensionality of input vectors.
            out_size (int): Dimensionality of output vectors.
            embed_size (int): Dimensionality of word embedding.
            hidden_size (int) : Dimensionality of hidden vectors.
            proj_size (int) : Dimensionality of projection before softmax.
            dropout (float): Dropout ratio.
        """
        super(LSTMDecoder, self).__init__(
            embed = L.EmbedID(in_size, embed_size),
            lstm = L.NStepLSTM(n_layers, embed_size, hidden_size, dropout),
            proj = L.Linear(hidden_size, proj_size),
            out = L.Linear(proj_size, out_size)
        )
        self.dropout = dropout
        for param in self.params():
            param.data[...] = np.random.uniform(-0.1, 0.1, param.data.shape) 

def forward(self, x, t):
        # def forward(self, x):
        h = F.max_pooling_2d(F.local_response_normalization(
            F.relu(self.conv1(x))), 3, stride=2)
        h = F.max_pooling_2d(F.local_response_normalization(
            F.relu(self.conv2(h))), 3, stride=2)
        h = F.relu(self.conv3(h))
        h = F.relu(self.conv4(h))
        h = F.max_pooling_2d(F.relu(self.conv5(h)), 3, stride=2)
        h = F.dropout(F.relu(self.fc6(h)))
        h = F.dropout(F.relu(self.fc7(h)))
        h = self.fc8(h)

        loss = F.softmax_cross_entropy(h, t)
        #loss = h

        # chainer.report({'loss': loss, 'accuracy': F.accuracy(h, t)}, self)
        return loss 

def __call__(self, x, t):
        h = F.relu(self.conv1(x))
        h = F.max_pooling_2d(h, 2, 1)
        h = F.relu(self.conv2(h))
        h = F.relu(self.conv3(h))
        h = F.dropout(F.relu(self.fc4(h)), train=self.train)
        h = self.fc5(h)
        h = F.reshape(h, (x.data.shape[0], 3, 16, 16))
        h = self.channelwise_inhibited(h)

        if self.train:
            self.loss = F.softmax_cross_entropy(h, t, normalize=False)
            return self.loss
        else:
            self.pred = F.softmax(h)
            return self.pred 

def __init__(self, n_layers, in_size, out_size, embed_size, hidden_size, proj_size, dropout=0.5):
        """Initialize encoder with structure parameters

        Args:
            n_layers (int): Number of layers.
            in_size (int): Dimensionality of input vectors.
            out_size (int): Dimensionality of output vectors.
            embed_size (int): Dimensionality of word embedding.
            hidden_size (int) : Dimensionality of hidden vectors.
            proj_size (int) : Dimensionality of projection before softmax.
            dropout (float): Dropout ratio.
        """
        super(LSTMDecoder, self).__init__(
            embed = L.EmbedID(in_size, embed_size),
            lstm = L.NStepLSTM(n_layers, embed_size, hidden_size, dropout),
            proj = L.Linear(hidden_size, proj_size),
            out = L.Linear(proj_size, out_size)
        )
        self.dropout = dropout
        for param in self.params():
            param.data[...] = np.random.uniform(-0.1, 0.1, param.data.shape) 

def __call__(self, prev_states, x_in):
        input_below = x_in
        states_cursor = 0
        res = []
        for i in six.moves.range(len(self)):
            if self.dropout is not None and not (self.no_dropout_on_input and i == 0):
                input_below = F.dropout(input_below, ratio=self.dropout)
            new_states = self[i](prev_states[states_cursor:states_cursor + self.nb_of_states[i]], input_below)
            states_cursor += self.nb_of_states[i]

            if (self.residual_connection and
                not (i == len(self) - 1 and self.no_residual_connection_on_output) and
                    not (i == 0 and self.no_residual_connection_on_input)):
                input_below = new_states[-1] + input_below
            else:
                input_below = new_states[-1]

            res += list(new_states)
        return res 

def forward(self, x, t):
        h = F.max_pooling_2d(F.local_response_normalization(
            F.relu(self.conv1(x))), 3, stride=2)
        h = F.max_pooling_2d(F.local_response_normalization(
            F.relu(self.conv2(h))), 3, stride=2)
        h = F.relu(self.conv3(h))
        h = F.relu(self.conv4(h))
        h = F.max_pooling_2d(F.relu(self.conv5(h)), 3, stride=2)
        h = F.dropout(F.relu(self.fc6(h)))
        h = F.dropout(F.relu(self.fc7(h)))
        h = self.fc8(h)

        # EDIT(hamaji): ONNX-chainer cannot output SoftmaxCrossEntropy.
        # loss = F.softmax_cross_entropy(h, t)
        loss = self.softmax_cross_entropy(h, t)
        if self.compute_accuracy:
            chainer.report({'loss': loss, 'accuracy': F.accuracy(h, t)}, self)
        else:
            chainer.report({'loss': loss}, self)
        return loss 

def __init__(self, d_model = 512, n_heads = 8, experimental_relu=False, dropout=None):
        if d_model%n_heads != 0:
            raise ValueError("d_model(%i) should be divisible by n_head(%i)"%(d_model, n_heads))
        
        super(ConstantSizeMultiBatchMultiHeadAttention, self).__init__(
            w_Q = L.Linear(d_model, d_model, nobias=False),
            w_K = L.Linear(d_model, d_model, nobias=True),
            w_V = L.Linear(d_model, d_model, nobias=False),
            )
        
        if n_heads >= 2:
            self.add_link("w_O", L.Linear(d_model, d_model)) #if n_heads == 1, it is redundant with w_V
        
        self.d_model = d_model
        self.n_heads = n_heads
        self.head_size = d_model // n_heads
        
        scaling_factor = 1.0 / self.xp.sqrt(self.xp.array([[[[self.head_size]]]], dtype=self.xp.float32))
        self.add_persistent("scaling_factor", scaling_factor) #added as persistent so that it works with to_gpu/to_cpu
        
        self.experimental_relu = experimental_relu
        
        self.dropout = dropout 

def __call__(self, seq_list):
        mb_size = len(seq_list)
        max_length_1 = max(len(x) for x in seq_list)
        x, mask = self.make_batch(seq_list)
    
        encoded = self.emb(x)
        encoded += self.get_pos_vect(mb_size, max_length_1)
        
        if self.dropout is not None:
            encoded = F.dropout(encoded, self.dropout)
        
        return self.encoding_layers(encoded, mask), mask 

def __call__(self, x):
        h = F.relu(self.conv1_1(x))
        h = F.relu(self.conv1_2(h))
        h = F.max_pooling_2d(h, 2, stride=2, pad=0)
        h = F.relu(self.conv2_1(h))
        h = F.relu(self.conv2_2(h))
        h = F.max_pooling_2d(h, 2, stride=2, pad=0)
        h = F.relu(self.conv3_1(h))
        h = F.relu(self.conv3_2(h))
        h = F.relu(self.conv3_3(h))
        pool3 = F.max_pooling_2d(h, 2, stride=2, pad=0)
        h = F.relu(self.conv4_1(pool3))
        h = F.relu(self.conv4_2(h))
        h = F.relu(self.conv4_3(h))
        pool4 = F.max_pooling_2d(h, 2, stride=2, pad=0)
        h = F.relu(self.conv5_1(pool4))
        h = F.relu(self.conv5_2(h))
        h = F.relu(self.conv5_3(h))
        h = F.max_pooling_2d(h, 2, stride=2, pad=0)
        h = F.relu(self.fc6(h))
        h = F.dropout(h, ratio=.5, train=self.train)
        h = F.relu(self.fc7(h))
        h = F.dropout(h, ratio=.5, train=self.train)
        score_fr = self.score_fr(h)

        upscore2 = self.upscore2(score_fr)
        score_pool4 = self.score_pool4(pool4)
        score_pool4c = f.crop_to_target(score_pool4, target=upscore2)
        fuse_pool4 = upscore2 + score_pool4c

        upscore_pool4 = self.upscore_pool4(fuse_pool4)
        score_pool3 = self.score_pool3(pool3)
        score_pool3c = f.crop_to_target(score_pool3, target=upscore_pool4)
        fuse_pool3 = upscore_pool4 + score_pool3c

        upscore8 = self.upscore8(fuse_pool3)
        score = f.crop_to_target(upscore8, target=x)

        return score 

def forward(self, x):
    x = F.relu(F.max_pooling_2d(self.conv1(x), 2))
    x = F.relu(F.max_pooling_2d(F.dropout(self.conv2(x)), 2))
    x = F.reshape(F.flatten(x), (-1, 320))
    x = F.relu(self.fc1(x))
    x = F.relu(self.fc2(x))
    return x 

def __call__(self, word, train=True):
        h1 = self.word_vec(word)
        h2 = self.lstm(F.dropout(h1, ratio=self.dropout_ratio))
        return self.out_word(F.dropout(h2, ratio=self.dropout_ratio)) 

def forward(self, x):
        y1 = F.dropout(x)
        return y1


# ======================================