Python transformers.BertModel() Examples
The following are 7
code examples of transformers.BertModel().
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example.
You may also want to check out all available functions/classes of the module
transformers
, or try the search function
.
.
Example #1
| Source File: sentence_encoder.py From hedwig with Apache License 2.0 | 5 votes |
|
def __init__(self, config):
super().__init__(config, num_labels=config.num_labels)
self.bert = BertModel(config)
self.dropout = nn.Dropout(config.hidden_dropout_prob)
self.init_weights()
Example #2
| Source File: modeling_bertabs.py From exbert with Apache License 2.0 | 5 votes |
|
def __init__(self):
super().__init__()
config = BertConfig.from_pretrained("bert-base-uncased")
self.model = BertModel(config)
Example #3
| Source File: modeling_bertabs.py From fast-bert with Apache License 2.0 | 5 votes |
|
def __init__(self):
super(Bert, self).__init__()
config = BertConfig.from_pretrained("bert-base-uncased")
self.model = BertModel(config)
Example #4
| Source File: BertCapModel.py From self-critical.pytorch with MIT License | 5 votes |
|
def make_model(self, src_vocab, tgt_vocab, N_enc=6, N_dec=6,
d_model=512, d_ff=2048, h=8, dropout=0.1):
"Helper: Construct a model from hyperparameters."
enc_config = BertConfig(vocab_size=1,
hidden_size=d_model,
num_hidden_layers=N_enc,
num_attention_heads=h,
intermediate_size=d_ff,
hidden_dropout_prob=dropout,
attention_probs_dropout_prob=dropout,
max_position_embeddings=1,
type_vocab_size=1)
dec_config = BertConfig(vocab_size=tgt_vocab,
hidden_size=d_model,
num_hidden_layers=N_dec,
num_attention_heads=h,
intermediate_size=d_ff,
hidden_dropout_prob=dropout,
attention_probs_dropout_prob=dropout,
max_position_embeddings=17,
type_vocab_size=1,
is_decoder=True)
encoder = BertModel(enc_config)
def return_embeds(*args, **kwargs):
return kwargs['inputs_embeds']
del encoder.embeddings; encoder.embeddings = return_embeds
decoder = BertModel(dec_config)
model = EncoderDecoder(
encoder,
decoder,
Generator(d_model, tgt_vocab))
return model
Example #5
| Source File: BertCapModel.py From ImageCaptioning.pytorch with MIT License | 5 votes |
|
def make_model(self, src_vocab, tgt_vocab, N_enc=6, N_dec=6,
d_model=512, d_ff=2048, h=8, dropout=0.1):
"Helper: Construct a model from hyperparameters."
enc_config = BertConfig(vocab_size=1,
hidden_size=d_model,
num_hidden_layers=N_enc,
num_attention_heads=h,
intermediate_size=d_ff,
hidden_dropout_prob=dropout,
attention_probs_dropout_prob=dropout,
max_position_embeddings=1,
type_vocab_size=1)
dec_config = BertConfig(vocab_size=tgt_vocab,
hidden_size=d_model,
num_hidden_layers=N_dec,
num_attention_heads=h,
intermediate_size=d_ff,
hidden_dropout_prob=dropout,
attention_probs_dropout_prob=dropout,
max_position_embeddings=17,
type_vocab_size=1,
is_decoder=True)
encoder = BertModel(enc_config)
def return_embeds(*args, **kwargs):
return kwargs['inputs_embeds']
del encoder.embeddings; encoder.embeddings = return_embeds
decoder = BertModel(dec_config)
model = EncoderDecoder(
encoder,
decoder,
Generator(d_model, tgt_vocab))
return model
Example #6
| Source File: transformers_embedder.py From DeepPavlov with Apache License 2.0 | 5 votes |
|
def load(self):
self.model = transformers.BertModel.from_pretrained(self.load_path, config=self.config).eval().to(self.device)
self.dim = self.model.config.hidden_size
Example #7
| Source File: absa_layer.py From BERT-E2E-ABSA with Apache License 2.0 | 4 votes |
|
def __init__(self, bert_config):
"""
:param bert_config: configuration for bert model
"""
super(BertABSATagger, self).__init__(bert_config)
self.num_labels = bert_config.num_labels
self.tagger_config = TaggerConfig()
self.tagger_config.absa_type = bert_config.absa_type.lower()
if bert_config.tfm_mode == 'finetune':
# initialized with pre-trained BERT and perform finetuning
# print("Fine-tuning the pre-trained BERT...")
self.bert = BertModel(bert_config)
else:
raise Exception("Invalid transformer mode %s!!!" % bert_config.tfm_mode)
self.bert_dropout = nn.Dropout(bert_config.hidden_dropout_prob)
# fix the parameters in BERT and regard it as feature extractor
if bert_config.fix_tfm:
# fix the parameters of the (pre-trained or randomly initialized) transformers during fine-tuning
for p in self.bert.parameters():
p.requires_grad = False
self.tagger = None
if self.tagger_config.absa_type == 'linear':
# hidden size at the penultimate layer
penultimate_hidden_size = bert_config.hidden_size
else:
self.tagger_dropout = nn.Dropout(self.tagger_config.hidden_dropout_prob)
if self.tagger_config.absa_type == 'lstm':
self.tagger = LSTM(input_size=bert_config.hidden_size,
hidden_size=self.tagger_config.hidden_size,
bidirectional=self.tagger_config.bidirectional)
elif self.tagger_config.absa_type == 'gru':
self.tagger = GRU(input_size=bert_config.hidden_size,
hidden_size=self.tagger_config.hidden_size,
bidirectional=self.tagger_config.bidirectional)
elif self.tagger_config.absa_type == 'tfm':
# transformer encoder layer
self.tagger = nn.TransformerEncoderLayer(d_model=bert_config.hidden_size,
nhead=12,
dim_feedforward=4*bert_config.hidden_size,
dropout=0.1)
elif self.tagger_config.absa_type == 'san':
# vanilla self attention networks
self.tagger = SAN(d_model=bert_config.hidden_size, nhead=12, dropout=0.1)
elif self.tagger_config.absa_type == 'crf':
self.tagger = CRF(num_tags=self.num_labels)
else:
raise Exception('Unimplemented downstream tagger %s...' % self.tagger_config.absa_type)
penultimate_hidden_size = self.tagger_config.hidden_size
self.classifier = nn.Linear(penultimate_hidden_size, bert_config.num_labels)
