Python chainer.links.LayerNormalization() Examples

def layer_normalization(x, gamma, beta, eps=1e-5):
    """Layer normalization.
    This function implements a "layer normalization"
    which normalizes the input units by statistics
    that are computed along the second axis,
    scales and shifts them.
    Args:
        x (~chainer.Variable): Batch vectors.
            Shape of this value must be `(batch_size, unit_size)`,
            e.g., the output of :func:`~chainer.functions.linear`.
        gamma (~chainer.Variable): Scaling vectors.
        beta (~chainer.Variable): Shifting vectors.
    Returns:
        ~chainer.Variable: The output variable which has the same shape
        as :math:`x`.
    See: `Layer Normalization <https://arxiv.org/abs/1607.06450>`_
    """
    return LayerNormalization(eps)(x, gamma, beta) 

def __init__(self, idim, n_layers, n_units,
                 e_units=2048, h=8, dropout=0.1):
        super(TransformerEncoder, self).__init__()
        with self.init_scope():
            self.linear_in = L.Linear(idim, n_units)
            self.lnorm_in = L.LayerNormalization(n_units)
            self.pos_enc = PositionalEncoding(n_units, dropout, 5000)
            self.n_layers = n_layers
            self.dropout = dropout
            for i in range(n_layers):
                setattr(self, '{}{:d}'.format("lnorm1_", i),
                        L.LayerNormalization(n_units))
                setattr(self, '{}{:d}'.format("self_att_", i),
                        MultiHeadSelfAttention(n_units, h))
                setattr(self, '{}{:d}'.format("lnorm2_", i),
                        L.LayerNormalization(n_units))
                setattr(self, '{}{:d}'.format("ff_", i),
                        PositionwiseFeedForward(n_units, e_units, dropout))
            self.lnorm_out = L.LayerNormalization(n_units) 

def __init__(self, dropout=None, residual_mode="normal", no_normalize=False):
        super(DropoutAndAddAndNormalize, self).__init__()
        
        if not no_normalize:
            LayerNormalization = get_layer_normalization_class()
            self.add_link("normalizing_layer", LayerNormalization())
        
        assert residual_mode in "normal none after".split()
        self.residual_mode = residual_mode
        self.no_normalize = no_normalize
        self.dropout = dropout 

def __init__(self, d_model=512, d_ff=2048, dropout=None,
            residual_mode="normal", no_normalize=False):
        super(FeedForward, self).__init__(
            lin1 = L.Linear(d_model, d_ff),
            lin2 = L.Linear(d_ff, d_model),
            layer_reduce = DropoutAndAddAndNormalize(dropout=dropout, residual_mode=residual_mode, no_normalize=no_normalize)
        )
        
#         self.dropout = dropout
#         
#         if not no_normalize:
#             self.add_link("normalization_layer", LayerNormalization())
#         
#         self.no_add = no_add
#         self.no_normalize = no_normalize 

def get_layer_normalization_class():
    global use_chainer_layer_normalization
    if use_chainer_layer_normalization:
        return L.LayerNormalization
    else:
        log.info("using faster LayerNormalization")
        return LayerNormalizationLink 

def _create_ln(*args, **kwargs):
    flag = chainer.disable_experimental_feature_warning
    chainer.disable_experimental_feature_warning = True
    try:
        return links.LayerNormalization(*args, **kwargs)
    finally:
        chainer.disable_experimental_feature_warning = flag 

def __init__(self, layer, size, dropout_ratio=0.1):
        super().__init__()
        self.dropout_ratio = dropout_ratio
        with self.init_scope():
            self.layer = layer
            self.norm = L.LayerNormalization(size) 

def __init__(self, sublayer, N):
        super().__init__()
        with self.init_scope():
            self.sub_layers = sublayer.repeat(N, mode='copy')
            self.norm = L.LayerNormalization(sublayer.size) 

def __init__(self, sublayer, N):
        super().__init__()
        with self.init_scope():
            self.sub_layers = sublayer.repeat(N, mode='copy')
            self.norm = L.LayerNormalization(sublayer.size) 

def __init__(self, layer, size, dropout_ratio=0.1):
        super().__init__()
        self.dropout_ratio = dropout_ratio
        with self.init_scope():
            self.layer = layer
            self.norm = L.LayerNormalization(size) 

def __init__(self, sublayer, N):
        super().__init__()
        with self.init_scope():
            self.sub_layers = sublayer.repeat(N, mode='copy')
            self.norm = L.LayerNormalization(sublayer.size) 

def __init__(self, sublayer, N):
        super().__init__()
        with self.init_scope():
            self.sub_layers = sublayer.repeat(N, mode='copy')
            self.norm = L.LayerNormalization(sublayer.size) 

def __init__(self, hidden_channels=16, n_edge_types=5,
                 activation=functions.relu):
        super(GNNFiLMUpdate, self).__init__()
        self.n_edge_types = n_edge_types
        self.activation = activation
        with self.init_scope():
            self.W_linear = GraphLinear(
                in_size=None, out_size=self.n_edge_types * hidden_channels,
                nobias=True)  # W_l in eq. (6)
            self.W_g = GraphLinear(
                in_size=None, out_size=self.n_edge_types * hidden_channels * 2,
                nobias=True)  # g in eq. (6)
            self.norm_layer = links.LayerNormalization()  # l in eq. (6) 

def __init__(self, ch0, ch1, \
                nn='conv', \
                norm='bn', \
                activation=F.relu, \
                dropout=False, \
                noise=None, \
                w_init=None, \
                k_size = 3, \
                normalize_input=False ):

        self.norm = norm
        self.normalize_input = normalize_input
        self.activation = activation
        self.dropout = dropout
        self.noise = noise
        self.nn = nn
        layers = {}

        if w_init == None:
            w = chainer.initializers.GlorotNormal()
        else:
            w = w_init

        if nn == 'down_conv':
            layers['c'] = L.Convolution2D(ch0, ch1, 4, 2, 1, initialW=w)

        elif nn == 'up_deconv':
            layers['c'] = L.Deconvolution2D(ch0, ch1, 4, 2, 1, initialW=w)

        elif nn == 'up_subpixel':
            pad = k_size//2
            layers['c'] = L.Convolution2D(ch0, ch1*4, k_size, 1, pad, initialW=w)

        elif nn=='conv' or nn=='up_unpooling':
            pad = k_size//2
            layers['c'] = L.Convolution2D(ch0, ch1, k_size, 1, pad, initialW=w)

        elif nn=='linear':
            layers['c'] = L.Linear(ch0, ch1, initialW=w)

        else:
            raise Exception("Cannot find method %s" % nn)

        if self.norm == 'bn':
            if self.noise:
                layers['n'] = L.BatchNormalization(ch1, use_gamma=False)
            else:
                layers['n'] = L.BatchNormalization(ch1)
        elif self.norm == 'ln':
                layers['n'] = L.LayerNormalization(ch1)

        super(NNBlock, self).__init__(**layers)