Python torch.nn.LSTMCell() Examples

def __init__(self,
                 num_feature_channels,
                 raymarch_steps):
        super().__init__()

        self.n_feature_channels = num_feature_channels
        self.steps = raymarch_steps

        hidden_size = 16
        self.lstm = nn.LSTMCell(input_size=self.n_feature_channels,
                                hidden_size=hidden_size)

        self.lstm.apply(init_recurrent_weights)
        lstm_forget_gate_init(self.lstm)

        self.out_layer = nn.Linear(hidden_size, 1)
        self.counter = 0 

def __init__(self, opt, use_maxout=False):
        super(VSUACore, self).__init__()
        self.opt = opt
        self.drop_prob_lm = opt.drop_prob_lm

        self.att_lstm = nn.LSTMCell(opt.input_encoding_size + opt.rnn_size * 2, opt.rnn_size) # we, fc, h^2_t-1

        lang_lstm_in_dim = opt.rnn_size*(1+len(self.opt.vsua_use))
        self.lang_lstm = nn.LSTMCell(lang_lstm_in_dim, opt.rnn_size) # h^1_t, \hat v

        if 'o' in self.opt.vsua_use:
            self.attention_obj = Attention(opt)
        if 'a' in self.opt.vsua_use:
            self.attention_attr = Attention(opt)
        if 'r' in self.opt.vsua_use:
            self.attention_rela = Attention(opt) 

def forward(self, x, hidden):
        # get prev_t, cell_t from states
        hx, cx = hidden
        Wx = F.linear(x, self.weight_ih)
        Uz = F.linear(hx, self.weight_hh)

        # Section 2.1 in https://arxiv.org/pdf/1606.06630.pdf
        gates = self.alpha * Wx * Uz + self.beta_i * Wx + self.beta_h * Uz + self.bias

        # Same as LSTMCell after this point
        ingate, forgetgate, cellgate, outgate = gates.chunk(4, 1)

        ingate = F.sigmoid(ingate)
        forgetgate = F.sigmoid(forgetgate)
        cellgate = F.tanh(cellgate)
        outgate = F.sigmoid(outgate)

        cy = (forgetgate * cx) + (ingate * cellgate)
        hy = outgate * F.tanh(cy)

        return hy, cy 

def __init__(self, input_size, hidden_size, num_layers=1,
                 dropout=0, bidirectional=False):
        super(LSTM, self).__init__()

        self.input_size = input_size
        self.hidden_size = hidden_size
        self.num_layers = num_layers
        self.dropout = dropout
        self.bidirectional = bidirectional
        self.num_directions = 2 if bidirectional else 1

        self.f_cells = nn.ModuleList()
        self.b_cells = nn.ModuleList()
        for layer in range(self.num_layers):
            self.f_cells.append(nn.LSTMCell(input_size=input_size,
                                            hidden_size=hidden_size))
            if bidirectional:
                self.b_cells.append(nn.LSTMCell(input_size=input_size,
                                                hidden_size=hidden_size))
            input_size = hidden_size * self.num_directions

        self.reset_parameters() 

def __init__(self, args):
        super(GraphWriter, self).__init__()
        self.args = args
        if args.title:
            self.title_emb = nn.Embedding(len(args.title_vocab), args.nhid, padding_idx=0)
            self.title_enc = BiLSTM(args, enc_type='title')
            self.title_attn = MSA(args)
        self.ent_emb = nn.Embedding(len(args.ent_text_vocab), args.nhid, padding_idx=0)
        self.tar_emb = nn.Embedding(len(args.text_vocab), args.nhid, padding_idx=0)
        if args.title:
            nn.init.xavier_normal_(self.title_emb.weight)
        nn.init.xavier_normal_(self.ent_emb.weight)
        self.rel_emb = nn.Embedding(len(args.rel_vocab), args.nhid, padding_idx=0)
        nn.init.xavier_normal_(self.rel_emb.weight)
        self.decode_lstm = nn.LSTMCell(args.dec_ninp, args.nhid)
        self.ent_enc = BiLSTM(args, enc_type='entity')
        self.graph_enc = GraphTrans(args)
        self.ent_attn = MSA(args)
        self.copy_attn = MSA(args, mode='copy')
        self.copy_fc = nn.Linear(args.dec_ninp, 1)
        self.pred_v_fc = nn.Linear(args.dec_ninp, len(args.text_vocab)) 

def forward(self, input, mask=None, hx=None):
        batch_size = input.size(0) if self.batch_first else input.size(1)
        lstm = self.Cell is nn.LSTMCell
        if hx is None:
            num_directions = 2 if self.bidirectional else 1
            hx = input.new_zeros(self.num_layers * num_directions, batch_size, self.hidden_size)
            if lstm:
                hx = (hx, hx)

        func = AutogradMaskedRNN(num_layers=self.num_layers,
                                 batch_first=self.batch_first,
                                 dropout=self.dropout,
                                 train=self.training,
                                 bidirectional=self.bidirectional,
                                 lstm=lstm)

        output, hidden = func(input, self.all_cells, hx, None if mask is None else mask.view(mask.size() + (1, )))
        return output, hidden 

def init_hidden(config, batch_size, cell):
    layers = 1
    if isinstance(cell, (nn.LSTM, nn.GRU)):
        layers = cell.num_layers
        if cell.bidirectional:
            layers = layers * 2

    if isinstance(cell, (nn.LSTM, nn.LSTMCell)):
        hidden  = Variable(torch.zeros(layers, batch_size, cell.hidden_size))
        context = Variable(torch.zeros(layers, batch_size, cell.hidden_size))
    
        if config.CONFIG.cuda:
            hidden  = hidden.cuda()
            context = context.cuda()
        return hidden, context

    if isinstance(cell, (nn.GRU, nn.GRUCell)):
        hidden  = Variable(torch.zeros(layers, batch_size, cell.hidden_size))
        if config.CONFIG.cuda:
            hidden  = hidden.cuda()
        return hidden 

def __init__(self, n_action, batch_size,
                 n_burn_in=40, nstep_return=5,
                 input_shape=(4, 84, 84)):
        super(DuelingLSTMDQN, self).__init__()
        self.n_action = n_action
        self.batch_size = batch_size
        self.n_burn_in = n_burn_in
        self.nstep_return = nstep_return
        self.input_shape = input_shape
        self.conv1 = nn.Conv2d(input_shape[0], 32, kernel_size=8, stride=4)
        self.conv2 = nn.Conv2d(32, 64, kernel_size=4, stride=2)
        self.conv3 = nn.Conv2d(64, 64, kernel_size=3, stride=1)
        r = int((int(input_shape[1] / 4) - 1) / 2) - 3
        c = int((int(input_shape[2] / 4) - 1) / 2) - 3
        self.lstm = nn.LSTMCell(r * c * 64, 512)
        self.adv1 = nn.Linear(512, 512)
        self.adv2 = nn.Linear(512, self.n_action, bias=False)
        self.val1 = nn.Linear(512, 512)
        self.val2 = nn.Linear(512, 1)
        self.hx = torch.zeros(self.batch_size, 512)
        self.cx = torch.zeros(self.batch_size, 512) 

def __init__(self, num_inputs, action_space, small_net=False):
        """
        Really I should be using inheritance for the small_net here
        """
        super(ES, self).__init__()
        num_outputs = action_space.n
        self.small_net = small_net
        if self.small_net:
            self.linear1 = nn.Linear(num_inputs, 64)
            self.linear2 = nn.Linear(64, 64)
            self.actor_linear = nn.Linear(64, num_outputs)
        else:
            self.conv1 = nn.Conv2d(num_inputs, 32, 3, stride=2, padding=1)
            self.conv2 = nn.Conv2d(32, 32, 3, stride=2, padding=1)
            self.conv3 = nn.Conv2d(32, 32, 3, stride=2, padding=1)
            self.conv4 = nn.Conv2d(32, 32, 3, stride=2, padding=1)
            self.lstm = nn.LSTMCell(32*3*3, 256)
            self.actor_linear = nn.Linear(256, num_outputs)
        self.train() 

def __init__(self, num_inputs, action_space):
        super(ActorCritic, self).__init__()
        self.conv1 = nn.Conv2d(num_inputs, 32, 3, stride=2, padding=1)
        self.conv2 = nn.Conv2d(32, 32, 3, stride=2, padding=1)
        self.conv3 = nn.Conv2d(32, 32, 3, stride=2, padding=1)
        self.conv4 = nn.Conv2d(32, 32, 3, stride=2, padding=1)

        self.lstm = nn.LSTMCell(32 * 3 * 3, 256)

        num_outputs = action_space.n
        self.critic_linear = nn.Linear(256, 1)
        self.actor_linear = nn.Linear(256, num_outputs)

        self.apply(weights_init)
        self.actor_linear.weight.data = normalized_columns_initializer(
            self.actor_linear.weight.data, 0.01)
        self.actor_linear.bias.data.fill_(0)
        self.critic_linear.weight.data = normalized_columns_initializer(
            self.critic_linear.weight.data, 1.0)
        self.critic_linear.bias.data.fill_(0)

        self.lstm.bias_ih.data.fill_(0)
        self.lstm.bias_hh.data.fill_(0)

        self.train() 

def __init__(self, hidden_size, layer_norm=False, input_gate=True, forget_gate=True):
            nn.Module.__init__(self)
            self.hidden_size = hidden_size
            # gradient(2), param(2), loss
            self.lstm = nn.LSTMCell(input_size=5, hidden_size=hidden_size)
            if layer_norm:
                self.layer_norm = nn.LayerNorm(hidden_size)
            else:
                self.layer_norm = None
            self.input_gate = input_gate
            self.forget_gate = forget_gate
            if self.input_gate:
                self.lr_layer = nn.Linear(hidden_size, 1)
                self.lrs = []
            else:
                self.output_layer = nn.Linear(hidden_size, 1)
                self.dets = []
            if forget_gate:
                self.fg_layer = nn.Linear(hidden_size, 1)
                self.fgs = []
            self.h_0 = nn.Parameter(torch.randn((hidden_size,), requires_grad=True))
            self.c_0 = nn.Parameter(torch.randn((hidden_size,), requires_grad=True)) 

def __init__(self, classes=4, bidirection = False, layernum=1, char_size = 69, hiddensize = 100):
        super(smallcharRNN, self).__init__()
        # self.embd = nn.Embedding(length, embedding_size)
        # self.lstm = nn.LSTMCell(hiddensize, hiddensize)
        self.lstm = nn.LSTM(char_size, hiddensize, layernum, bidirectional = bidirection)
        self.hiddensize = hiddensize
        numdirections = 1 + bidirection
        self.hsize = numdirections * layernum
        self.linear = nn.Linear(hiddensize * numdirections, classes)
        self.log_softmax = nn.LogSoftmax() 

def step(self, input, hx=None, mask=None):
        '''
        execute one step forward (only for one-directional RNN).
        Args:
            input (batch, input_size): input tensor of this step.
            hx (num_layers, batch, hidden_size): the hidden state of last step.
            mask (batch): the mask tensor of this step.

        Returns:
            output (batch, hidden_size): tensor containing the output of this step from the last layer of RNN.
            hn (num_layers, batch, hidden_size): tensor containing the hidden state of this step
        '''
        assert not self.bidirectional, "step only cannot be applied to bidirectional RNN."
        batch_size = input.size(0)
        lstm = self.Cell is nn.LSTMCell
        if hx is None:
            hx = input.new_zeros(self.num_layers, batch_size, self.hidden_size)
            if lstm:
                hx = (hx, hx)

        func = AutogradMaskedStep(num_layers=self.num_layers,
                                 dropout=self.dropout,
                                 train=self.training,
                                 lstm=lstm)

        output, hidden = func(input, self.all_cells, hx, mask)
        return output, hidden 

def __init__(self, opt, use_maxout=False):
        super(TopDownCore, self).__init__()
        self.drop_prob_lm = opt.drop_prob_lm

        self.att_lstm = nn.LSTMCell(opt.input_encoding_size + opt.rnn_size * 2, opt.rnn_size) # we, fc, h^2_t-1
        self.lang_lstm = nn.LSTMCell(opt.rnn_size * 2, opt.rnn_size) # h^1_t, \hat v
        self.attention = Attention(opt) 

def __init__(self, opt):
        super(AATCore, self).__init__()
        
        self.drop_prob_lm = opt.drop_prob_lm
        self.rnn_size = opt.rnn_size
        self.epsilon = opt.epsilon
        self.max_att_steps = opt.max_att_steps
        self.use_multi_head = opt.use_multi_head

        self.att_lstm = nn.LSTMCell(opt.input_encoding_size + opt.rnn_size, opt.rnn_size)
        
        self.confidence = nn.Sequential(nn.Linear(opt.rnn_size, opt.rnn_size),
                                    nn.ReLU(),
                                    nn.Linear(opt.rnn_size, 1),
                                    nn.Sigmoid())

        self.h2query = nn.Sequential(nn.Linear(opt.rnn_size * 2, opt.rnn_size),
                            nn.ReLU())

        # if opt.use_multi_head == 1: # TODO, not implemented for now           
        #     self.attention = MultiHeadedAddAttention(opt.num_heads, opt.d_model, scale=opt.multi_head_scale)
        if opt.use_multi_head == 2:
            self.attention = MultiHeadedDotAttention(opt.num_heads, opt.rnn_size, project_k_v=0, scale=opt.multi_head_scale, use_output_layer=0, do_aoa=0, norm_q=1)
        else:            
            self.attention = Attention(opt)

        self.lang_lstm = nn.LSTMCell(opt.rnn_size + opt.rnn_size, opt.rnn_size)

        self.norm_h = LayerNorm(opt.rnn_size)
        self.norm_c = LayerNorm(opt.rnn_size) 

def __init__(self, opt):
        super(AoA_Decoder_Core, self).__init__()
        self.drop_prob_lm = opt.drop_prob_lm
        self.d_model = opt.rnn_size
        self.use_multi_head = opt.use_multi_head
        self.multi_head_scale = opt.multi_head_scale
        self.use_ctx_drop = getattr(opt, 'ctx_drop', 0)
        self.out_res = getattr(opt, 'out_res', 0)
        self.decoder_type = getattr(opt, 'decoder_type', 'AoA')
        self.att_lstm = nn.LSTMCell(opt.input_encoding_size + opt.rnn_size, opt.rnn_size) # we, fc, h^2_t-1
        self.out_drop = nn.Dropout(self.drop_prob_lm)

        if self.decoder_type == 'AoA':
            # AoA layer
            self.att2ctx = nn.Sequential(nn.Linear(self.d_model * opt.multi_head_scale + opt.rnn_size, 2 * opt.rnn_size), nn.GLU())
        elif self.decoder_type == 'LSTM':
            # LSTM layer
            self.att2ctx = nn.LSTMCell(self.d_model * opt.multi_head_scale + opt.rnn_size, opt.rnn_size)
        else:
            # Base linear layer
            self.att2ctx = nn.Sequential(nn.Linear(self.d_model * opt.multi_head_scale + opt.rnn_size, opt.rnn_size), nn.ReLU())

        # if opt.use_multi_head == 1: # TODO, not implemented for now           
        #     self.attention = MultiHeadedAddAttention(opt.num_heads, opt.d_model, scale=opt.multi_head_scale)
        if opt.use_multi_head == 2:            
            self.attention = MultiHeadedDotAttention(opt.num_heads, opt.rnn_size, project_k_v=0, scale=opt.multi_head_scale, use_output_layer=0, do_aoa=0, norm_q=1)
        else:            
            self.attention = Attention(opt)

        if self.use_ctx_drop:
            self.ctx_drop = nn.Dropout(self.drop_prob_lm)        
        else:
            self.ctx_drop = lambda x :x 

def __init__(self, attention_dim, embed_dim, decoder_dim, vocab_size, encoder_dim=2048, dropout=0.5):
        """
        :param attention_dim: size of attention network
        :param embed_dim: embedding size
        :param decoder_dim: size of decoder's RNN
        :param vocab_size: size of vocabulary
        :param encoder_dim: feature size of encoded images
        :param dropout: dropout
        """
        super(DecoderWithAttention, self).__init__()

        self.encoder_dim = encoder_dim
        self.attention_dim = attention_dim
        self.embed_dim = embed_dim
        self.decoder_dim = decoder_dim
        self.vocab_size = vocab_size
        self.dropout = dropout

        self.attention = Attention(encoder_dim, decoder_dim, attention_dim)  # attention network

        self.embedding = nn.Embedding(vocab_size, embed_dim)  # embedding layer
        self.dropout = nn.Dropout(p=self.dropout)
        self.decode_step = nn.LSTMCell(embed_dim + encoder_dim, decoder_dim, bias=True)  # decoding LSTMCell
        self.init_h = nn.Linear(encoder_dim, decoder_dim)  # linear layer to find initial hidden state of LSTMCell
        self.init_c = nn.Linear(encoder_dim, decoder_dim)  # linear layer to find initial cell state of LSTMCell
        self.f_beta = nn.Linear(decoder_dim, encoder_dim)  # linear layer to create a sigmoid-activated gate
        self.sigmoid = nn.Sigmoid()
        self.fc = nn.Linear(decoder_dim, vocab_size)  # linear layer to find scores over vocabulary
        self.init_weights()  # initialize some layers with the uniform distribution 

def __init__(self, opt, use_maxout=False):
        super(TopDownCore, self).__init__()
        self.drop_prob_lm = opt.drop_prob_lm
        self.min_value = -1e8

        self.att_lstm = nn.LSTMCell(opt.input_encoding_size + opt.rnn_size, opt.rnn_size) # we, fc, h^2_t-1

        # self.att_lstm = nn.LSTMCell(opt.input_encoding_size + opt.rnn_size * 2, opt.rnn_size) # we, fc, h^2_t-1
        self.lang_lstm = nn.LSTMCell(opt.rnn_size*2, opt.rnn_size) # h^1_t, \hat v
        self.attention = Attention(opt)
        self.attention2 = Attention2(opt)

        self.adaPnt = adaPnt(opt.input_encoding_size, opt.rnn_size, opt.att_hid_size, self.drop_prob_lm, self.min_value, opt.beta)
        self.i2h_2 = nn.Linear(opt.rnn_size*2, opt.rnn_size)
        self.h2h_2 = nn.Linear(opt.rnn_size, opt.rnn_size) 

def __init__(self, args):
        super(LSTMController, self).__init__(args)

        # build model
        self.in_2_hid = nn.LSTMCell(self.input_dim + self.read_vec_dim, self.hidden_dim, 1)

        self._reset() 

def __init__(self, num_layers, input_size, rnn_size, dropout):
        super(StackedLSTMCell, self).__init__()
        self.dropout = nn.Dropout(dropout)
        self.num_layers = num_layers

        self.layers = nn.ModuleList()
        for i in range(num_layers):
            self.layers.append(nn.LSTMCell(input_size, rnn_size))
            input_size = rnn_size 

def __init__(self, classes=4, bidirection = False, layernum=1, length=20000,embedding_size =100, hiddensize = 100):
        super(smallRNN, self).__init__()
        self.embd = nn.Embedding(length, embedding_size)
        # self.lstm = nn.LSTMCell(hiddensize, hiddensize)
        self.lstm = nn.LSTM(embedding_size, hiddensize, layernum, bidirectional = bidirection)
        self.hiddensize = hiddensize
        numdirections = 1 + bidirection
        self.hsize = numdirections * layernum
        self.linear = nn.Linear(hiddensize * numdirections, classes)
        self.log_softmax = nn.LogSoftmax() 

def __init__(self, classes=4, bidirection = False, layernum=1, char_size = 69, hiddensize = 100):
        super(smallcharRNN, self).__init__()
        # self.embd = nn.Embedding(length, embedding_size)
        # self.lstm = nn.LSTMCell(hiddensize, hiddensize)
        self.lstm = nn.LSTM(char_size, hiddensize, layernum, bidirectional = bidirection)
        self.hiddensize = hiddensize
        numdirections = 1 + bidirection
        self.hsize = numdirections * layernum
        self.linear = nn.Linear(hiddensize * numdirections, classes)
        self.log_softmax = nn.LogSoftmax() 

def __init__(self, classes=4, bidirection = False, layernum=1, length=20000,embedding_size =100, hiddensize = 100):
        super(smallRNN, self).__init__()
        self.embd = nn.Embedding(length, embedding_size)
        # self.lstm = nn.LSTMCell(hiddensize, hiddensize)
        self.lstm = nn.LSTM(embedding_size, hiddensize, layernum, bidirectional = bidirection)
        self.hiddensize = hiddensize
        numdirections = 1 + bidirection
        self.hsize = numdirections * layernum
        self.linear = nn.Linear(hiddensize * numdirections, classes)
        self.log_softmax = nn.LogSoftmax() 

def test_cuda_nn(self):
        # These throw if cuda is misconfigured
        tnn.GRUCell(10,10).cuda()
        tnn.RNNCell(10,10).cuda()
        tnn.LSTMCell(10,10).cuda()
        tnn.GRU(10,10).cuda()
        tnn.LSTM(10,10).cuda()
        tnn.RNN(10,10).cuda() 

def __init__(self, embed_size, state_size, out_size):
        super().__init__()
        self.net_lstm = nn.LSTMCell(embed_size, state_size)
        self.net_mlp = nn.Linear(state_size, out_size)
        self.softmax = nn.Softmax()
        xavier_init(self) 

def forward(self, x, h0, x_mask=None, h_mask=None):
        h0 = self.query_wsum(h0, h_mask)
        if type(self.rnn) is nn.LSTMCell:
            c0 = Variable(h0.new(h0.size()).zero_())
        scores_list = []
        for turn in range(self.num_turn):
            att_scores = self.attn(x, h0, x_mask)
            x_sum = torch.bmm(F.softmax(att_scores, 1).unsqueeze(1), x).squeeze(1)
            scores = self.classifier(x_sum, h0)
            scores_list.append(scores)
            # next turn
            if self.rnn is not None:
                h0 = self.dropout(h0)
                if type(self.rnn) is nn.LSTMCell:
                    h0, c0 = self.rnn(x_sum, (h0, c0))
                else:
                    h0 = self.rnn(x_sum, h0)
        if self.mem_type == 1:
            mask = generate_mask(self.alpha.data.new(x.size(0), self.num_turn), self.mem_random_drop, self.training)
            mask = [m.contiguous() for m in torch.unbind(mask, 1)]
            tmp_scores_list = [mask[idx].view(x.size(0), 1).expand_as(inp) * F.softmax(inp, 1) for idx, inp in enumerate(scores_list)]
            scores = torch.stack(tmp_scores_list, 2)
            scores = torch.mean(scores, 2)
            scores = torch.log(scores)
        else:
            scores = scores_list[-1]
        if self.dump_state:
            return scores, scores_list
        else:
            return scores 

def __init__(self, node_manifestations, hidden=100):
    super(_SearchSpaceManifestation, self).__init__()
    self.lstm = nn.LSTMCell(hidden, hidden)
    self.prev_attention = []
    self.node_manifestations = node_manifestations 

def _initialize_weights(self):
        for module in self.modules():
            if isinstance(module, nn.Conv2d) or isinstance(module, nn.Linear):
                nn.init.xavier_uniform_(module.weight)
                # nn.init.kaiming_uniform_(module.weight)
                nn.init.constant_(module.bias, 0)
            elif isinstance(module, nn.LSTMCell):
                nn.init.constant_(module.bias_ih, 0)
                nn.init.constant_(module.bias_hh, 0) 

def __init__(self, *args, **kwargs):
        r"""
        
        :param input_size:  输入 `x` 的特征维度
        :param hidden_size: 隐状态  `h`  的特征维度
        :param num_layers: rnn的层数. Default: 1
        :param bias: 如果为 ``False``, 模型将不会使用bias. Default: ``True``
        :param batch_first: 若为 ``True``, 输入和输出 ``Tensor`` 形状为
            (batch, seq, feature). Default: ``False``
        :param input_dropout: 对输入的dropout概率. Default: 0
        :param hidden_dropout: 对每个隐状态的dropout概率. Default: 0
        :param bidirectional: 若为 ``True``, 使用双向的LSTM. Default: ``False``
        """
        super(VarLSTM, self).__init__(
            mode="LSTM", Cell=nn.LSTMCell, *args, **kwargs) 

def __init__(self, latents, actions, hiddens, gaussians):
        super().__init__(latents, actions, hiddens, gaussians)
        self.rnn = nn.LSTMCell(latents + actions, hiddens)