Python tensorflow.contrib.slim.fully_connected() Examples
The following are 30
code examples of tensorflow.contrib.slim.fully_connected().
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example.
You may also want to check out all available functions/classes of the module
tensorflow.contrib.slim
, or try the search function
.
Example #1
Source File: dfc_vae_large.py From TNT with GNU General Public License v3.0 | 6 votes |
def encoder(self, images, is_training): activation_fn = leaky_relu # tf.nn.relu weight_decay = 0.0 with tf.variable_scope('encoder'): with slim.arg_scope([slim.batch_norm], is_training=is_training): with slim.arg_scope([slim.conv2d, slim.fully_connected], weights_initializer=tf.truncated_normal_initializer(stddev=0.1), weights_regularizer=slim.l2_regularizer(weight_decay), normalizer_fn=slim.batch_norm, normalizer_params=self.batch_norm_params): net = slim.conv2d(images, 32, [4, 4], 2, activation_fn=activation_fn, scope='Conv2d_1') net = slim.conv2d(net, 64, [4, 4], 2, activation_fn=activation_fn, scope='Conv2d_2') net = slim.conv2d(net, 128, [4, 4], 2, activation_fn=activation_fn, scope='Conv2d_3') net = slim.conv2d(net, 256, [4, 4], 2, activation_fn=activation_fn, scope='Conv2d_4') net = slim.conv2d(net, 512, [4, 4], 2, activation_fn=activation_fn, scope='Conv2d_5') net = slim.flatten(net) fc1 = slim.fully_connected(net, self.latent_variable_dim, activation_fn=None, normalizer_fn=None, scope='Fc_1') fc2 = slim.fully_connected(net, self.latent_variable_dim, activation_fn=None, normalizer_fn=None, scope='Fc_2') return fc1, fc2
Example #2
Source File: hidden_combine_chain_model.py From youtube-8m with Apache License 2.0 | 6 votes |
def create_model(self, model_input, vocab_size, num_mixtures=None, l2_penalty=1e-8, sub_scope="", original_input=None, **unused_params): num_supports = FLAGS.num_supports num_layers = FLAGS.hidden_chain_layers relu_cells = FLAGS.hidden_chain_relu_cells next_input = model_input support_predictions = [] for layer in xrange(num_layers): sub_relu = slim.fully_connected( next_input, relu_cells, activation_fn=tf.nn.relu, weights_regularizer=slim.l2_regularizer(l2_penalty), scope=sub_scope+"relu-%d"%layer) sub_prediction = self.sub_model(sub_relu, vocab_size, sub_scope=sub_scope+"prediction-%d"%layer) relu_norm = tf.nn.l2_normalize(sub_relu, dim=1) next_input = tf.concat([next_input, relu_norm], axis=1) support_predictions.append(sub_prediction) main_predictions = self.sub_model(next_input, vocab_size, sub_scope=sub_scope+"-main") support_predictions = tf.concat(support_predictions, axis=1) return {"predictions": main_predictions, "support_predictions": support_predictions}
Example #3
Source File: inception_resnet_v1.py From TNT with GNU General Public License v3.0 | 6 votes |
def inference(images, keep_probability, phase_train=True, bottleneck_layer_size=128, weight_decay=0.0, reuse=None): batch_norm_params = { # Decay for the moving averages. 'decay': 0.995, # epsilon to prevent 0s in variance. 'epsilon': 0.001, # force in-place updates of mean and variance estimates 'updates_collections': None, # Moving averages ends up in the trainable variables collection 'variables_collections': [ tf.GraphKeys.TRAINABLE_VARIABLES ], } with slim.arg_scope([slim.conv2d, slim.fully_connected], weights_initializer=slim.initializers.xavier_initializer(), weights_regularizer=slim.l2_regularizer(weight_decay), normalizer_fn=slim.batch_norm, normalizer_params=batch_norm_params): return inception_resnet_v1(images, is_training=phase_train, dropout_keep_prob=keep_probability, bottleneck_layer_size=bottleneck_layer_size, reuse=reuse)
Example #4
Source File: dummy.py From TNT with GNU General Public License v3.0 | 6 votes |
def inference(images, keep_probability, phase_train=True, # @UnusedVariable bottleneck_layer_size=128, bottleneck_layer_activation=None, weight_decay=0.0, reuse=None): # @UnusedVariable batch_norm_params = { # Decay for the moving averages. 'decay': 0.995, # epsilon to prevent 0s in variance. 'epsilon': 0.001, # force in-place updates of mean and variance estimates 'updates_collections': None, # Moving averages ends up in the trainable variables collection 'variables_collections': [ tf.GraphKeys.TRAINABLE_VARIABLES ], } with slim.arg_scope([slim.conv2d, slim.fully_connected], weights_initializer=tf.truncated_normal_initializer(stddev=0.1), weights_regularizer=slim.l2_regularizer(weight_decay), normalizer_fn=slim.batch_norm, normalizer_params=batch_norm_params): size = np.prod(images.get_shape()[1:].as_list()) net = slim.fully_connected(tf.reshape(images, (-1,size)), bottleneck_layer_size, activation_fn=None, scope='Bottleneck', reuse=False) return net, None
Example #5
Source File: mobilenet_v2.py From R2CNN_Faster-RCNN_Tensorflow with MIT License | 6 votes |
def mobilenetv2_scope(is_training=True, trainable=True, weight_decay=0.00004, stddev=0.09, dropout_keep_prob=0.8, bn_decay=0.997): """Defines Mobilenet training scope. In default. We do not use BN ReWrite the scope. """ batch_norm_params = { 'is_training': False, 'trainable': False, 'decay': bn_decay, } with slim.arg_scope(training_scope(is_training=is_training, weight_decay=weight_decay)): with slim.arg_scope([slim.conv2d, slim.fully_connected, slim.separable_conv2d], trainable=trainable): with slim.arg_scope([slim.batch_norm], **batch_norm_params) as sc: return sc
Example #6
Source File: dfc_vae.py From TNT with GNU General Public License v3.0 | 6 votes |
def encoder(self, images, is_training): activation_fn = leaky_relu # tf.nn.relu weight_decay = 0.0 with tf.variable_scope('encoder'): with slim.arg_scope([slim.batch_norm], is_training=is_training): with slim.arg_scope([slim.conv2d, slim.fully_connected], weights_initializer=tf.truncated_normal_initializer(stddev=0.1), weights_regularizer=slim.l2_regularizer(weight_decay), normalizer_fn=slim.batch_norm, normalizer_params=self.batch_norm_params): net = slim.conv2d(images, 32, [4, 4], 2, activation_fn=activation_fn, scope='Conv2d_1') net = slim.conv2d(net, 64, [4, 4], 2, activation_fn=activation_fn, scope='Conv2d_2') net = slim.conv2d(net, 128, [4, 4], 2, activation_fn=activation_fn, scope='Conv2d_3') net = slim.conv2d(net, 256, [4, 4], 2, activation_fn=activation_fn, scope='Conv2d_4') net = slim.flatten(net) fc1 = slim.fully_connected(net, self.latent_variable_dim, activation_fn=None, normalizer_fn=None, scope='Fc_1') fc2 = slim.fully_connected(net, self.latent_variable_dim, activation_fn=None, normalizer_fn=None, scope='Fc_2') return fc1, fc2
Example #7
Source File: model.py From minimal-entropy-correlation-alignment with MIT License | 6 votes |
def E(self, images, is_training = False, reuse=False): if images.get_shape()[3] == 3: images = tf.image.rgb_to_grayscale(images) with tf.variable_scope('encoder',reuse=reuse): with slim.arg_scope([slim.fully_connected], activation_fn=tf.nn.relu): with slim.arg_scope([slim.conv2d], activation_fn=tf.nn.relu, padding='VALID'): net = slim.conv2d(images, 64, 5, scope='conv1') net = slim.max_pool2d(net, 2, stride=2, scope='pool1') net = slim.conv2d(net, 128, 5, scope='conv2') net = slim.max_pool2d(net, 2, stride=2, scope='pool2') net = tf.contrib.layers.flatten(net) net = slim.fully_connected(net, 1024, activation_fn=tf.nn.relu, scope='fc3') net = slim.dropout(net, 0.5, is_training=is_training) net = slim.fully_connected(net, self.hidden_repr_size, activation_fn=tf.tanh,scope='fc4') # dropout here or not? #~ net = slim.dropout(net, 0.5, is_training=is_training) return net
Example #8
Source File: nasnet_model.py From benchmarks with Apache License 2.0 | 6 votes |
def _build_aux_head(net, end_points, num_classes, hparams, scope): """Auxiliary head used for all models across all datasets.""" with tf.variable_scope(scope): aux_logits = tf.identity(net) with tf.variable_scope('aux_logits'): aux_logits = slim.avg_pool2d( aux_logits, [5, 5], stride=3, padding='VALID') aux_logits = slim.conv2d(aux_logits, 128, [1, 1], scope='proj') aux_logits = slim.batch_norm(aux_logits, scope='aux_bn0') aux_logits = tf.nn.relu(aux_logits) # Shape of feature map before the final layer. shape = aux_logits.shape if hparams.data_format == 'NHWC': shape = shape[1:3] else: shape = shape[2:4] aux_logits = slim.conv2d(aux_logits, 768, shape, padding='VALID') aux_logits = slim.batch_norm(aux_logits, scope='aux_bn1') aux_logits = tf.nn.relu(aux_logits) aux_logits = contrib_layers.flatten(aux_logits) aux_logits = slim.fully_connected(aux_logits, num_classes) end_points['AuxLogits'] = aux_logits
Example #9
Source File: hidden_chain_model.py From youtube-8m with Apache License 2.0 | 6 votes |
def create_model(self, model_input, vocab_size, num_mixtures=None, l2_penalty=1e-8, sub_scope="", original_input=None, **unused_params): num_supports = FLAGS.num_supports num_layers = FLAGS.hidden_chain_layers relu_cells = FLAGS.hidden_chain_relu_cells next_input = model_input support_predictions = [] for layer in xrange(num_layers): sub_relu = slim.fully_connected( next_input, relu_cells, activation_fn=tf.nn.relu, weights_regularizer=slim.l2_regularizer(l2_penalty), scope=sub_scope+"relu-%d"%layer) sub_prediction = self.sub_model(sub_relu, vocab_size, sub_scope=sub_scope+"prediction-%d"%layer) relu_norm = tf.nn.l2_normalize(sub_relu, dim=1) next_input = tf.concat([model_input, relu_norm], axis=1) support_predictions.append(sub_prediction) main_predictions = self.sub_model(next_input, vocab_size, sub_scope=sub_scope+"-main") support_predictions = tf.concat(support_predictions, axis=1) return {"predictions": main_predictions, "support_predictions": support_predictions}
Example #10
Source File: squeezenet.py From tf_ctpn with MIT License | 6 votes |
def _arg_scope(self, is_training, reuse=None): weight_decay = 0.0 keep_probability = 1.0 batch_norm_params = { 'is_training': is_training, # Decay for the moving averages. 'decay': 0.995, # epsilon to prevent 0s in variance. 'epsilon': 0.001 } with slim.arg_scope([slim.conv2d, slim.fully_connected], weights_initializer=slim.xavier_initializer_conv2d(uniform=True), weights_regularizer=slim.l2_regularizer(weight_decay), normalizer_fn=slim.batch_norm, normalizer_params=batch_norm_params): with tf.variable_scope(self._scope, self._scope, reuse=reuse): with slim.arg_scope([slim.batch_norm, slim.dropout], is_training=is_training) as sc: return sc
Example #11
Source File: pyramid_network.py From FastMaskRCNN with Apache License 2.0 | 6 votes |
def _extra_conv_arg_scope(weight_decay=0.00001, activation_fn=None, normalizer_fn=None): with slim.arg_scope( [slim.conv2d, slim.conv2d_transpose], padding='SAME', weights_regularizer=slim.l2_regularizer(weight_decay), weights_initializer=tf.truncated_normal_initializer(stddev=0.001), activation_fn=activation_fn, normalizer_fn=normalizer_fn,) as arg_sc: with slim.arg_scope( [slim.fully_connected], weights_regularizer=slim.l2_regularizer(weight_decay), weights_initializer=tf.truncated_normal_initializer(stddev=0.001), activation_fn=activation_fn, normalizer_fn=normalizer_fn) as arg_sc: return arg_sc
Example #12
Source File: mobilenet_v2.py From R2CNN-Plus-Plus_Tensorflow with MIT License | 6 votes |
def mobilenetv2_scope(is_training=True, trainable=True, weight_decay=0.00004, stddev=0.09, dropout_keep_prob=0.8, bn_decay=0.997): """Defines Mobilenet training scope. In default. We do not use BN ReWrite the scope. """ batch_norm_params = { 'is_training': False, 'trainable': False, 'decay': bn_decay, } with slim.arg_scope(training_scope(is_training=is_training, weight_decay=weight_decay)): with slim.arg_scope([slim.conv2d, slim.fully_connected, slim.separable_conv2d], trainable=trainable): with slim.arg_scope([slim.batch_norm], **batch_norm_params) as sc: return sc
Example #13
Source File: attention.py From R2CNN-Plus-Plus_Tensorflow with MIT License | 6 votes |
def squeeze_excitation_layer(input_x, out_dim, ratio, layer_name, is_training): with tf.name_scope(layer_name): # Global_Average_Pooling squeeze = tf.reduce_mean(input_x, [1, 2]) excitation = slim.fully_connected(inputs=squeeze, num_outputs=out_dim // ratio, weights_initializer=cfgs.BBOX_INITIALIZER, activation_fn=tf.nn.relu, trainable=is_training, scope=layer_name+'_fully_connected1') excitation = slim.fully_connected(inputs=excitation, num_outputs=out_dim, weights_initializer=cfgs.BBOX_INITIALIZER, activation_fn=tf.nn.sigmoid, trainable=is_training, scope=layer_name + '_fully_connected2') excitation = tf.reshape(excitation, [-1, 1, 1, out_dim]) # scale = input_x * excitation return excitation
Example #14
Source File: mobilenet_v2.py From R3Det_Tensorflow with MIT License | 6 votes |
def mobilenetv2_scope(is_training=True, trainable=True, weight_decay=0.00004, stddev=0.09, dropout_keep_prob=0.8, bn_decay=0.997): """Defines Mobilenet training scope. In default. We do not use BN ReWrite the scope. """ batch_norm_params = { 'is_training': False, 'trainable': False, 'decay': bn_decay, } with slim.arg_scope(training_scope(is_training=is_training, weight_decay=weight_decay)): with slim.arg_scope([slim.conv2d, slim.fully_connected, slim.separable_conv2d], trainable=trainable): with slim.arg_scope([slim.batch_norm], **batch_norm_params) as sc: return sc
Example #15
Source File: mobilenet_v2_r3det_plusplus.py From R3Det_Tensorflow with MIT License | 6 votes |
def mobilenetv2_scope(is_training=True, trainable=True, weight_decay=0.00004, stddev=0.09, dropout_keep_prob=0.8, bn_decay=0.997): """Defines Mobilenet training scope. In default. We do not use BN ReWrite the scope. """ batch_norm_params = { 'is_training': False, 'trainable': False, 'decay': bn_decay, } with slim.arg_scope(training_scope(is_training=is_training, weight_decay=weight_decay)): with slim.arg_scope([slim.conv2d, slim.fully_connected, slim.separable_conv2d], trainable=trainable): with slim.arg_scope([slim.batch_norm], **batch_norm_params) as sc: return sc
Example #16
Source File: input_moe_model.py From youtube-8m with Apache License 2.0 | 6 votes |
def create_model(self, model_input, vocab_size, num_mixtures=None, l2_penalty=1e-8, sub_scope="", original_input=None, **unused_params): num_methods = model_input.get_shape().as_list()[-1] num_features = model_input.get_shape().as_list()[-2] original_input = tf.nn.l2_normalize(original_input, dim=1) gate_activations = slim.fully_connected( original_input, num_methods, activation_fn=tf.nn.softmax, weights_regularizer=slim.l2_regularizer(l2_penalty), scope="gates"+sub_scope) output = tf.einsum("ijk,ik->ij", model_input, gate_activations) return {"predictions": output}
Example #17
Source File: inception_resnet_v2.py From TNT with GNU General Public License v3.0 | 6 votes |
def inference(images, keep_probability, phase_train=True, bottleneck_layer_size=128, weight_decay=0.0, reuse=None): batch_norm_params = { # Decay for the moving averages. 'decay': 0.995, # epsilon to prevent 0s in variance. 'epsilon': 0.001, # force in-place updates of mean and variance estimates 'updates_collections': None, # Moving averages ends up in the trainable variables collection 'variables_collections': [ tf.GraphKeys.TRAINABLE_VARIABLES ], } with slim.arg_scope([slim.conv2d, slim.fully_connected], weights_initializer=slim.initializers.xavier_initializer(), weights_regularizer=slim.l2_regularizer(weight_decay), normalizer_fn=slim.batch_norm, normalizer_params=batch_norm_params): return inception_resnet_v2(images, is_training=phase_train, dropout_keep_prob=keep_probability, bottleneck_layer_size=bottleneck_layer_size, reuse=reuse)
Example #18
Source File: multitask_divergence_moe_model.py From youtube-8m with Apache License 2.0 | 5 votes |
def sub_model(self, model_input, vocab_size, num_mixtures=None, l2_penalty=1e-8, sub_scope="", dropout=False, keep_prob=None, noise_level=None, **unused_params): num_mixtures = num_mixtures or FLAGS.moe_num_mixtures if dropout: model_input = tf.nn.dropout(model_input, keep_prob=keep_prob) gate_activations = slim.fully_connected( model_input, vocab_size * (num_mixtures + 1), activation_fn=None, biases_initializer=None, weights_regularizer=slim.l2_regularizer(l2_penalty), scope="gates-"+sub_scope) expert_activations = slim.fully_connected( model_input, vocab_size * num_mixtures, activation_fn=None, weights_regularizer=slim.l2_regularizer(l2_penalty), scope="experts-"+sub_scope) gating_distribution = tf.nn.softmax(tf.reshape( gate_activations, [-1, num_mixtures + 1])) # (Batch * #Labels) x (num_mixtures + 1) expert_distribution = tf.nn.sigmoid(tf.reshape( expert_activations, [-1, num_mixtures])) # (Batch * #Labels) x num_mixtures final_probabilities_by_class_and_batch = tf.reduce_sum( gating_distribution[:, :num_mixtures] * expert_distribution, 1) final_probabilities = tf.reshape(final_probabilities_by_class_and_batch, [-1, vocab_size]) return final_probabilities
Example #19
Source File: deep_combine_chain_model.py From youtube-8m with Apache License 2.0 | 5 votes |
def create_model(self, model_input, vocab_size, num_mixtures=None, l2_penalty=1e-8, sub_scope="", original_input=None, dropout=False, keep_prob=None, noise_level=None, num_frames=None, **unused_params): num_supports = FLAGS.num_supports num_layers = FLAGS.deep_chain_layers relu_cells = FLAGS.deep_chain_relu_cells relu_type = FLAGS.deep_chain_relu_type use_length = FLAGS.deep_chain_use_length next_input = model_input support_predictions = [] for layer in xrange(num_layers): sub_prediction = self.sub_model(next_input, vocab_size, sub_scope=sub_scope+"prediction-%d"%layer, dropout=dropout, keep_prob=keep_prob, noise_level=noise_level) sub_activation = slim.fully_connected( sub_prediction, relu_cells, activation_fn=None, weights_regularizer=slim.l2_regularizer(l2_penalty), scope=sub_scope+"relu-%d"%layer) if relu_type == "elu": sub_relu = tf.nn.elu(sub_activation) else: # default: relu sub_relu = tf.nn.relu(sub_activation) if noise_level is not None: print "adding noise to sub_relu, level = ", noise_level sub_relu = sub_relu + tf.random_normal(tf.shape(sub_relu), mean=0.0, stddev=noise_level) relu_norm = tf.nn.l2_normalize(sub_relu, dim=1) next_input = tf.concat([next_input, relu_norm], axis=1) support_predictions.append(sub_prediction) main_predictions = self.sub_model(next_input, vocab_size, sub_scope=sub_scope+"-main") support_predictions = tf.concat(support_predictions, axis=1) return {"predictions": main_predictions, "support_predictions": support_predictions}
Example #20
Source File: video_level_models.py From youtube-8m with Apache License 2.0 | 5 votes |
def create_model(self, model_input, vocab_size, l2_penalty=1e-8, **unused_params): """Creates a logistic model. Args: model_input: 'batch' x 'num_features' matrix of input features. vocab_size: The number of classes in the dataset. Returns: A dictionary with a tensor containing the probability predictions of the model in the 'predictions' key. The dimensions of the tensor are batch_size x num_classes.""" output = slim.fully_connected( model_input, vocab_size, activation_fn=tf.nn.sigmoid, weights_regularizer=slim.l2_regularizer(l2_penalty)) return {"predictions": output}
Example #21
Source File: distillchain_deep_combine_chain_model.py From youtube-8m with Apache License 2.0 | 5 votes |
def sub_model(self, model_input, vocab_size, num_mixtures=None, l2_penalty=1e-8, sub_scope="", dropout=False, keep_prob=None, noise_level=None, **unused_params): num_mixtures = num_mixtures or FLAGS.moe_num_mixtures if dropout: model_input = tf.nn.dropout(model_input, keep_prob=keep_prob) gate_activations = slim.fully_connected( model_input, vocab_size * (num_mixtures + 1), activation_fn=None, biases_initializer=None, weights_regularizer=slim.l2_regularizer(l2_penalty), scope="gates-"+sub_scope) expert_activations = slim.fully_connected( model_input, vocab_size * num_mixtures, activation_fn=None, weights_regularizer=slim.l2_regularizer(l2_penalty), scope="experts-"+sub_scope) gating_distribution = tf.nn.softmax(tf.reshape( gate_activations, [-1, num_mixtures + 1])) # (Batch * #Labels) x (num_mixtures + 1) expert_distribution = tf.nn.sigmoid(tf.reshape( expert_activations, [-1, num_mixtures])) # (Batch * #Labels) x num_mixtures final_probabilities_by_class_and_batch = tf.reduce_sum( gating_distribution[:, :num_mixtures] * expert_distribution, 1) final_probabilities = tf.reshape(final_probabilities_by_class_and_batch, [-1, vocab_size]) return final_probabilities
Example #22
Source File: deep_chain_model.py From youtube-8m with Apache License 2.0 | 5 votes |
def create_model(self, model_input, vocab_size, num_mixtures=None, l2_penalty=1e-8, sub_scope="", original_input=None, num_frames=None, **unused_params): num_supports = FLAGS.num_supports num_layers = FLAGS.deep_chain_layers relu_cells = FLAGS.deep_chain_relu_cells use_length = FLAGS.deep_chain_use_length if use_length: model_input = tf.concat([model_input, self.get_length_code(num_frames)], axis=1) next_input = model_input support_predictions = [] for layer in xrange(num_layers): sub_prediction = self.sub_model(next_input, vocab_size, sub_scope=sub_scope+"prediction-%d"%layer) sub_relu = slim.fully_connected( sub_prediction, relu_cells, activation_fn=tf.nn.relu, weights_regularizer=slim.l2_regularizer(l2_penalty), scope=sub_scope+"relu-%d"%layer) relu_norm = tf.nn.l2_normalize(sub_relu, dim=1) next_input = tf.concat([model_input, relu_norm], axis=1) support_predictions.append(sub_prediction) main_predictions = self.sub_model(next_input, vocab_size, sub_scope=sub_scope+"-main") support_predictions = tf.concat(support_predictions, axis=1) return {"predictions": main_predictions, "support_predictions": support_predictions}
Example #23
Source File: framehop_lstm_memory_deep_combine_chain_model.py From youtube-8m with Apache License 2.0 | 5 votes |
def sub_model(self, model_input, vocab_size, num_mixtures=None, l2_penalty=1e-8, sub_scope="", dropout=False, keep_prob=None, noise_level=None, **unused_params): num_mixtures = num_mixtures or FLAGS.moe_num_mixtures if dropout: print "adding dropout to moe, keep_prob = ", keep_prob model_input = tf.nn.dropout(model_input, keep_prob=keep_prob) gate_activations = slim.fully_connected( model_input, vocab_size * (num_mixtures + 1), activation_fn=None, biases_initializer=None, weights_regularizer=slim.l2_regularizer(l2_penalty), scope="gates-"+sub_scope) expert_activations = slim.fully_connected( model_input, vocab_size * num_mixtures, activation_fn=None, weights_regularizer=slim.l2_regularizer(l2_penalty), scope="experts-"+sub_scope) gating_distribution = tf.nn.softmax(tf.reshape( gate_activations, [-1, num_mixtures + 1])) # (Batch * #Labels) x (num_mixtures + 1) expert_distribution = tf.nn.sigmoid(tf.reshape( expert_activations, [-1, num_mixtures])) # (Batch * #Labels) x num_mixtures final_probabilities_by_class_and_batch = tf.reduce_sum( gating_distribution[:, :num_mixtures] * expert_distribution, 1) final_probabilities = tf.reshape(final_probabilities_by_class_and_batch, [-1, vocab_size]) return final_probabilities
Example #24
Source File: chain_main_relu_moe_model.py From youtube-8m with Apache License 2.0 | 5 votes |
def create_model(self, model_input, vocab_size, num_mixtures=None, l2_penalty=1e-8, sub_scope="", original_input=None, **unused_params): num_supports = FLAGS.num_supports input_size = model_input.shape.as_list()[1] support_predictions = self.sub_model(model_input, num_supports, sub_scope=sub_scope+"-support") main_relu = slim.fully_connected( model_input, input_size, activation_fn=tf.nn.relu, weights_regularizer=slim.l2_regularizer(l2_penalty), scope="main-relu-"+sub_scope) main_input = tf.concat([main_relu, support_predictions], axis=1) main_predictions = self.sub_model(main_input, vocab_size, sub_scope=sub_scope+"-main") return {"predictions": main_predictions, "support_predictions": support_predictions}
Example #25
Source File: logistic_model.py From youtube-8m with Apache License 2.0 | 5 votes |
def create_model(self, model_input, vocab_size, l2_penalty=1e-8, original_input=None, **unused_params): """Creates a logistic model. Args: model_input: 'batch' x 'num_features' matrix of input features. vocab_size: The number of classes in the dataset. Returns: A dictionary with a tensor containing the probability predictions of the model in the 'predictions' key. The dimensions of the tensor are batch_size x num_classes.""" output = slim.fully_connected( model_input, vocab_size, activation_fn=tf.nn.sigmoid, weights_regularizer=slim.l2_regularizer(l2_penalty)) return {"predictions": output}
Example #26
Source File: labels_embedding.py From youtube-8m with Apache License 2.0 | 5 votes |
def create_model(self, model_input, vocab_size, l2_penalty=1e-8, **unused_params): """Creates a logistic model. Args: model_input: 'batch' x 'num_features' matrix of input features. vocab_size: The number of classes in the dataset. Returns: A dictionary with a tensor containing the probability predictions of the model in the 'predictions' key. The dimensions of the tensor are batch_size x num_classes.""" model_input = tf.cast(model_input,dtype=tf.float32) hidden_size = FLAGS.hidden_size model_mask, indices_input = tf.nn.top_k(model_input, k=FLAGS.top_k) indices_input = tf.reshape(indices_input, [-1]) models_mask = tf.reshape(model_mask, [-1,FLAGS.top_k,1]) with tf.name_scope("embedding"): embeddings = tf.Variable( tf.random_uniform([vocab_size, hidden_size], -1.0, 1.0)) embed = tf.nn.embedding_lookup(embeddings, indices_input) output = slim.fully_connected( embed, vocab_size, activation_fn=tf.nn.sigmoid, weights_regularizer=slim.l2_regularizer(l2_penalty), scope="output") indices_one_hot = tf.one_hot(indices_input, vocab_size) output = output * (1 - indices_one_hot) + indices_one_hot output_val = tf.reshape(output,[-1,FLAGS.top_k,vocab_size]) predictions_val = tf.reduce_sum(output_val*models_mask, axis=1)/tf.reduce_sum(models_mask, axis=1) return {"predictions": output, "predictions_val": predictions_val}
Example #27
Source File: attention_moe_model.py From youtube-8m with Apache License 2.0 | 5 votes |
def relu(self, model_input, relu_cells, l2_penalty=1e-8, sub_scope=""): sub_activation = slim.fully_connected( model_input, relu_cells, activation_fn=tf.nn.relu, weights_regularizer=slim.l2_regularizer(l2_penalty), scope="relu-"+sub_scope) return sub_activation
Example #28
Source File: attention_moe_model.py From youtube-8m with Apache License 2.0 | 5 votes |
def create_model(self, model_input, vocab_size, num_mixtures=None, l2_penalty=1e-8, sub_scope="", original_input=None, **unused_params): num_relu = FLAGS.attention_relu_cells num_methods = model_input.get_shape().as_list()[-1] num_features = model_input.get_shape().as_list()[-2] original_input = tf.nn.l2_normalize(original_input, dim=1) model_input_list = tf.unstack(model_input, axis=2) relu_units = [self.relu(original_input, num_relu, sub_scope="input")] i = 0 for mi in model_input_list: relu_units.append(self.relu(mi, num_relu, sub_scope="sub"+str(i))) i += 1 gate_activations = slim.fully_connected( tf.concat(relu_units, axis=1), num_methods, activation_fn=None, biases_initializer=None, weights_regularizer=slim.l2_regularizer(l2_penalty), scope="gate") gate = tf.nn.softmax(gate_activations) output = tf.einsum("ijk,ik->ij", model_input, gate) return {"predictions": output}
Example #29
Source File: attention_linear_model.py From youtube-8m with Apache License 2.0 | 5 votes |
def create_model(self, model_input, vocab_size, num_mixtures=None, l2_penalty=1e-8, sub_scope="", original_input=None, **unused_params): num_methods = model_input.get_shape().as_list()[-1] num_features = model_input.get_shape().as_list()[-2] num_mixtures = FLAGS.moe_num_mixtures # gating coefficients original_input = tf.nn.l2_normalize(original_input, dim=1) mean_output = tf.reduce_mean(model_input, axis=2) ## batch_size x moe_num_mixtures gate_activations = slim.fully_connected( tf.concat([original_input, mean_output], axis=1), num_mixtures, activation_fn=tf.nn.softmax, weights_regularizer=slim.l2_regularizer(l2_penalty), scope="gates"+sub_scope) # matrix weight_var = tf.get_variable("ensemble_weight", shape=[num_mixtures, num_methods], regularizer=slim.l2_regularizer(l2_penalty)) # weight gated_weight = tf.einsum("ij,jk->ik", gate_activations, weight_var) weight = tf.nn.softmax(gated_weight) # weighted output output = tf.einsum("ik,ijk->ij", weight, model_input) return {"predictions": output}
Example #30
Source File: logistic_model.py From youtube-8m with Apache License 2.0 | 5 votes |
def create_model(self, model_input, vocab_size, l2_penalty=1e-8, original_input=None, **unused_params): """Creates a logistic model. Args: model_input: 'batch' x 'num_features' matrix of input features. vocab_size: The number of classes in the dataset. Returns: A dictionary with a tensor containing the probability predictions of the model in the 'predictions' key. The dimensions of the tensor are batch_size x num_classes.""" output = slim.fully_connected( model_input, vocab_size, activation_fn=tf.nn.sigmoid, weights_regularizer=slim.l2_regularizer(l2_penalty)) return {"predictions": output}