Python torch.nn.functional.layer_norm() Examples

The following are 8 code examples of torch.nn.functional.layer_norm(). You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may also want to check out all available functions/classes of the module torch.nn.functional , or try the search function .
Example #1
Source File: layer_norm.py    From fairseq with MIT License 5 votes vote down vote up 
def forward(self, input):
        output = F.layer_norm(
            input.float(),
            self.normalized_shape,
            self.weight.float() if self.weight is not None else None,
            self.bias.float() if self.bias is not None else None,
            self.eps,
        )
        return output.type_as(input)
Example #2
Source File: network.py    From DMIT with MIT License 5 votes vote down vote up 
def forward(self, x):
    normalized_shape = x.size()[1:]
    if self.affine:
      return F.layer_norm(x, normalized_shape, self.weight.expand(normalized_shape), self.bias.expand(normalized_shape))
    else:
      return F.layer_norm(x, normalized_shape)
Example #3
Source File: normalization.py    From pytorch-meta with MIT License 5 votes vote down vote up 
def forward(self, input, params=None):
        if params is None:
            params = OrderedDict(self.named_parameters())
        weight = params.get('weight', None)
        bias = params.get('bias', None)
        return F.layer_norm(
            input, self.normalized_shape, weight, bias, self.eps)
Example #4
Source File: layer_norm.py    From attn2d with MIT License 5 votes vote down vote up 
def forward(self, input):
        output = F.layer_norm(
            input.float(),
            self.normalized_shape,
            self.weight.float() if self.weight is not None else None,
            self.bias.float() if self.bias is not None else None,
            self.eps,
        )
        return output.type_as(input)
Example #5
Source File: deq_transformer.py    From deq with MIT License 5 votes vote down vote up 
def forward(self, inp, attn_out=None):
        assert inp.size(1) == self.d_model, "Feature dimension not match!!"

        if self.pre_lnorm:
            inp = F.layer_norm(inp.transpose(1,2), (self.d_model,)).transpose(1,2)
        relu_out1 = self.drop1(F.relu(self.ff1_net(inp)))
        out2 = self.drop2(self.ff2_net(relu_out1))
        output = out2 + inp
        if not self.pre_lnorm:
            output = F.layer_norm(output.transpose(1,2), (self.d_model,)).transpose(1,2)
        return output
Example #6
Source File: fused_layer_norm.py    From apex with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up 
def forward(self, input):
        if not input.is_cuda:
            return  F.layer_norm(
                input, self.normalized_shape, self.weight, self.bias, self.eps)
        if self.elementwise_affine:
          return FusedLayerNormAffineFunction.apply(
              input, self.weight, self.bias, self.normalized_shape,self.eps)
        else:
          return FusedLayerNormFunction.apply(input, self.normalized_shape, self.eps)
Example #7
Source File: test_pyprof_nvtx.py    From apex with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up 
def test_layer_norm(self):
        inp = torch.randn(1, 3, 32, 32, device='cuda', dtype=self.dtype)
        output = F.layer_norm(inp, inp.size()[1:], weight=None, bias=None, eps=1e-05)
Example #8
Source File: networks_pono.py    From PONO with MIT License 5 votes vote down vote up 
def forward(self, x):
        normalized_shape = x.size()[1:]
        if self.affine:
            return F.layer_norm(x, normalized_shape, self.weight.expand(normalized_shape), self.bias.expand(normalized_shape))
        else:
            return F.layer_norm(x, normalized_shape)