Python keras.backend.l2_normalize() Examples

def call(self, input_tensor, mask=None):
        z_s = input_tensor[0]
        z_n = input_tensor[1]
        r_s = input_tensor[2]

        z_s = K.l2_normalize(z_s, axis=-1)
        z_n = K.l2_normalize(z_n, axis=-1)
        r_s = K.l2_normalize(r_s, axis=-1)

        steps = z_n.shape[1]

        pos = K.sum(z_s * r_s, axis=-1, keepdims=True)
        pos = K.repeat_elements(pos, steps, axis=1)
        r_s = K.expand_dims(r_s, axis=-2)
        r_s = K.repeat_elements(r_s, steps, axis=1)
        neg = K.sum(z_n * r_s, axis=-1)

        loss = K.cast(K.sum(K.maximum(0., (1. - pos + neg)), axis=-1, keepdims=True), K.floatx())
        return loss 

def call(self, x, mask=None):
        uit = K.tanh(K.dot(x, self.Ws1))
        ait = K.dot(uit, self.Ws2)
        ait = K.permute_dimensions(ait, (0, 2, 1))
        A = K.softmax(ait, axis=1)
        M = K.batch_dot(A, x)
        if self.punish:
            A_T = K.permute_dimensions(A, (0, 2, 1))
            tile_eye = K.tile(K.eye(self.weight_ws2), [self.batch_size, 1])
            tile_eye = K.reshape(
                tile_eye, shape=[-1, self.weight_ws2, self.weight_ws2])
            AA_T = K.batch_dot(A, A_T) - tile_eye
            P = K.l2_normalize(AA_T, axis=(1, 2))
            return M, P
        else:
            return M 

def call(self, inputs):  # pylint:disable=arguments-differ
        """This is where the layer's logic lives.

        Parameters
        ----------
        inputs: tensor
            Input tensor, or list/tuple of input tensors
        kwargs: dict
            Additional keyword arguments

        Returns
        -------
        tensor
            A tensor or list/tuple of tensors
        """
        return K.l2_normalize(inputs, self.axis) 

def call(self, x):
        # feat : bz x W x H x D, cluster_score: bz X W x H x clusters.
        feat, cluster_score = x
        num_features = feat.shape[-1]

        # softmax normalization to get soft-assignment.
        # A : bz x W x H x clusters
        max_cluster_score = K.max(cluster_score, -1, keepdims=True)
        exp_cluster_score = K.exp(cluster_score - max_cluster_score)
        A = exp_cluster_score / K.sum(exp_cluster_score, axis=-1, keepdims = True)

        # Now, need to compute the residual, self.cluster: clusters x D
        A = K.expand_dims(A, -1)    # A : bz x W x H x clusters x 1
        feat_broadcast = K.expand_dims(feat, -2)    # feat_broadcast : bz x W x H x 1 x D
        feat_res = feat_broadcast - self.cluster    # feat_res : bz x W x H x clusters x D
        weighted_res = tf.multiply(A, feat_res)     # weighted_res : bz x W x H x clusters x D
        cluster_res = K.sum(weighted_res, [1, 2])

        if self.mode == 'gvlad':
            cluster_res = cluster_res[:, :self.k_centers, :]

        cluster_l2 = K.l2_normalize(cluster_res, -1)
        outputs = K.reshape(cluster_l2, [-1, int(self.k_centers) * int(num_features)])
        return outputs 

def _compute_energy(self, stm):
        # "concat" energy function
        # energy_i = g * V / |V| * tanh([stm, h_i] * W + b) + r
        _stm = K.dot(stm, self.W_a)

        V_a = self.V_a
        if self.normalize_energy:
            V_a = self.Energy_g * K.l2_normalize(self.V_a)

        et = K.dot(activations.tanh(K.expand_dims(_stm, axis=1) + self._uxpb),
                   K.expand_dims(V_a))

        if self.is_monotonic:
            et += self.Energy_r

        return et 

def get_model_41(params):
    embedding_weights = pickle.load(open("../data/datasets/train_data/embedding_weights_w2v-google_MSD-AG.pk","rb"))
    # main sequential model
    model = Sequential()
    model.add(Embedding(len(embedding_weights[0]), params['embedding_dim'], input_length=params['sequence_length'],
                        weights=embedding_weights))
    #model.add(Dropout(params['dropout_prob'][0], input_shape=(params['sequence_length'], params['embedding_dim'])))
    model.add(LSTM(2048))
    #model.add(Dropout(params['dropout_prob'][1]))
    model.add(Dense(output_dim=params["n_out"], init="uniform"))
    model.add(Activation(params['final_activation']))
    logging.debug("Output CNN: %s" % str(model.output_shape))

    if params['final_activation'] == 'linear':
        model.add(Lambda(lambda x :K.l2_normalize(x, axis=1)))

    return model


# CRNN Arch for audio 

def build_siamese_resnet_18(input_shape, num_outputs):
        channels, height, width = input_shape
        branch_channels = 3 #channels / 2
        branch_input_shape = (branch_channels, height, width)
        branch = ResnetBuilder.build_resnet_18(branch_input_shape, NUM_EMBEDDING, False)
        input = Input(shape=(height, width, channels))
        first_branch = branch(Lambda(lambda x: x[:, :, :, :3])(input))
        second_branch = branch(Lambda(lambda x: x[:, :, :, 3:])(input))
        if NORMALIZATION_ON:
            first_branch = Lambda(lambda x: K.l2_normalize(x, axis=1))(first_branch)
            second_branch = Lambda(lambda x: K.l2_normalize(x, axis=1))(second_branch) 
        
        raw_result = concatenate([first_branch, second_branch])
        output = _top_network(raw_result)
        
        # raw_result = dot([first_branch, second_branch], axes=1)
        # result = Lambda(lambda x: (K.clip(x, 0.5, 1) - 0.5) * 2.0)(raw_result)
        # negated_result = Lambda(lambda x: 1 - x)(result)
        # output = concatenate([negated_result, result])
        
        return Model(inputs=input, outputs=output) 

def call(self, inputs):
        x1 = inputs[0]
        x2 = inputs[1]
        if self.match_type in ['dot']:
            if self.normalize:
                x1 = K.l2_normalize(x1, axis=2)
                x2 = K.l2_normalize(x2, axis=2)
            output = K.tf.einsum('abd,acd->abc', x1, x2)
            output = K.tf.expand_dims(output, 3)
        elif self.match_type in ['mul', 'plus', 'minus']:
            x1_exp = K.tf.stack([x1] * self.shape2[1], 2)
            x2_exp = K.tf.stack([x2] * self.shape1[1], 1)
            if self.match_type == 'mul':
                output = x1_exp * x2_exp
            elif self.match_type == 'plus':
                output = x1_exp + x2_exp
            elif self.match_type == 'minus':
                output = x1_exp - x2_exp
        elif self.match_type in ['concat']:
            x1_exp = K.tf.stack([x1] * self.shape2[1], axis=2)
            x2_exp = K.tf.stack([x2] * self.shape1[1], axis=1)
            output = K.tf.concat([x1_exp, x2_exp], axis=3)

        return output 

def call(self, inputs: list, **kwargs) -> typing.Any:
        """
        The computation logic of MatchingTensorLayer.

        :param inputs: two input tensors.
        """
        x1 = inputs[0]
        x2 = inputs[1]
        # Normalize x1 and x2
        if self._normalize:
            x1 = K.l2_normalize(x1, axis=2)
            x2 = K.l2_normalize(x2, axis=2)

        # b = batch size
        # l = length of `x1`
        # r = length of `x2`
        # d, e = embedding size
        # c = number of channels
        # output = [b, c, l, r]
        output = tf.einsum(
            'bld,cde,bre->bclr',
            x1, self.interaction_matrix, x2
        )
        return output 

def call(self, inputs):
        inputs -= K.mean(inputs, axis=1, keepdims=True)
        inputs = K.l2_normalize(inputs, axis=1)
        pos = K.relu(inputs)
        neg = K.relu(-inputs)
        return K.concatenate([pos, neg], axis=1)

# global parameters 

def get_config(self):
        config = {
            'output_dim': self.output_dim,
            'activation': activations.serialize(self.activation),
            'use_bias': self.use_bias,
            'l2_normalize': self.l2_normalize,
            'output_raw_logits' : self.output_raw_logits,
        }
        base_config = super(HadamardClassifier, self).get_config()
        return dict(list(base_config.items()) + list(config.items())) 

def call(self, x, training=None):
        is_training = training not in {0, False}
        output = K.l2_normalize(x, axis=-1) if self.l2_normalize else x
        output = -self.scale * K.dot(output, self.hadamard) # pity .dot requires both tensors to be same type, the last one could be int8
        if self.output_raw_logits:
            output_logits = -self.scale * K.dot(x, self.hadamard) # probably better to reuse output * l2norm
        if self.use_bias:
            output = K.bias_add(output, self.bias)
            if self.output_raw_logits:
                output_logits = K.bias_add(output_logits, self.bias)
        if self.activation is not None:
            output = self.activation(output)
        if self.output_raw_logits:
            return [output, output_logits]
        return output 

def __init__(self, output_dim, activation=None, use_bias=True, 
                 l2_normalize=True, output_raw_logits=False, **kwargs):
        self.output_dim        = output_dim
        self.activation        = activations.get(activation)
        self.use_bias          = use_bias
        self.l2_normalize      = l2_normalize
        self.output_raw_logits = output_raw_logits

        super(HadamardClassifier, self).__init__(**kwargs) 

def call(self, inputs, **kwargs):
        # get cosine similarity
        # cosine = x * w / (||x|| * ||w||)
        inputs = K.l2_normalize(inputs, axis=1)
        kernel = K.l2_normalize(self.kernel, axis=0)
        cosine = K.dot(inputs, kernel)
        return cosine 

def call(self, inputs):
        inputs -= K.mean(inputs, axis=1, keepdims=True)
        inputs = K.l2_normalize(inputs, axis=1)
        pos = K.relu(inputs)
        neg = K.relu(-inputs)
        return K.concatenate([pos, neg], axis=1)

# global parameters 

def call(self, inputs):
        inputs -= K.mean(inputs, axis=1, keepdims=True)
        inputs = K.l2_normalize(inputs, axis=1)
        pos = K.relu(inputs)
        neg = K.relu(-inputs)
        return K.concatenate([pos, neg], axis=1)

# global parameters 

def call(self, inputs):
        inputs -= K.mean(inputs, axis=1, keepdims=True)
        inputs = K.l2_normalize(inputs, axis=1)
        pos = K.relu(inputs)
        neg = K.relu(-inputs)
        return K.concatenate([pos, neg], axis=1)

# global parameters 

def call(self, inputs):
        inputs -= K.mean(inputs, axis=1, keepdims=True)
        inputs = K.l2_normalize(inputs, axis=1)
        pos = K.relu(inputs)
        neg = K.relu(-inputs)
        return K.concatenate([pos, neg], axis=1)

# global parameters 

def call(self, inputs):
        inputs -= K.mean(inputs, axis=1, keepdims=True)
        inputs = K.l2_normalize(inputs, axis=1)
        pos = K.relu(inputs)
        neg = K.relu(-inputs)
        return K.concatenate([pos, neg], axis=1)

# global parameters 

def masked_spectral_distance(true, pred):
    # Note, fragment ions that cannot exists (i.e. y20 for a 7mer) must have the value  -1.
    import tensorflow
    import keras.backend as k

    epsilon = k.epsilon()
    pred_masked = ((true + 1) * pred) / (true + 1 + epsilon)
    true_masked = ((true + 1) * true) / (true + 1 + epsilon)
    pred_norm = k.l2_normalize(true_masked, axis=-1)
    true_norm = k.l2_normalize(pred_masked, axis=-1)
    product = k.sum(pred_norm * true_norm, axis=1)
    arccos = tensorflow.acos(product)
    return 2 * arccos / numpy.pi 

def build_tpe(n_in, n_out, W_pca=None):
    a = Input(shape=(n_in,))
    p = Input(shape=(n_in,))
    n = Input(shape=(n_in,))

    if W_pca is None:
        W_pca = np.zeros((n_in, n_out))

    base_model = Sequential()
    base_model.add(Dense(n_out, input_dim=n_in, bias=False, weights=[W_pca], activation='linear'))
    base_model.add(Lambda(lambda x: K.l2_normalize(x, axis=1)))

    # base_model = Sequential()
    # base_model.add(Dense(178, input_dim=n_in, bias=True, activation='relu'))
    # base_model.add(Dense(n_out, bias=True, activation='tanh'))
    # base_model.add(Lambda(lambda x: K.l2_normalize(x, axis=1)))

    a_emb = base_model(a)
    p_emb = base_model(p)
    n_emb = base_model(n)

    e = merge([a_emb, p_emb, n_emb], mode=triplet_merge, output_shape=triplet_merge_shape)

    model = Model(input=[a, p, n], output=e)
    predict = Model(input=a, output=a_emb)

    model.compile(loss=triplet_loss, optimizer='rmsprop')

    return model, predict 

def call(self, x, mask=None):
        output = K.l2_normalize(x, self.axis)
        return output * self.gamma 

def compute_similarity(self, tensor_1, tensor_2):
        return K.sum(K.l2_normalize(tensor_1, axis=-1) * K.l2_normalize(tensor_2, axis=-1),
                     axis=-1) 

def test_compute_similarity_does_a_cosine_similarity(self):
        a_vectors = numpy.asarray([[numpy.random.random(3) for _ in range(2)]], dtype="float32")
        b_vectors = numpy.asarray([[numpy.random.random(3) for _ in range(2)]], dtype="float32")
        normed_a = K.l2_normalize(K.variable(a_vectors), axis=-1)
        normed_b = K.l2_normalize(K.variable(b_vectors), axis=-1)
        desired_result = K.eval(self.dot_product.compute_similarity(normed_a, normed_b))
        result = K.eval(self.cosine_similarity.compute_similarity(K.variable(a_vectors), K.variable(b_vectors)))
        assert result.shape == (1, 2)    # batch_size = 1
        assert numpy.all(result == desired_result) 

def test_compute_similarity_works_with_higher_order_tensors(self):
        a_vectors = numpy.random.rand(5, 4, 3, 6, 7)
        b_vectors = numpy.random.rand(5, 4, 3, 6, 7)
        normed_a = K.eval(K.l2_normalize(K.variable(a_vectors), axis=-1))
        normed_b = K.eval(K.l2_normalize(K.variable(b_vectors), axis=-1))
        result = K.eval(self.cosine_similarity.compute_similarity(K.variable(a_vectors), K.variable(b_vectors)))
        assert result.shape == (5, 4, 3, 6)
        assert_almost_equal(result[3, 2, 1, 3],
                            numpy.dot(normed_a[3, 2, 1, 3], normed_b[3, 2, 1, 3]),
                            decimal=6) 

def build_bottom_network(edge_model, input_shape):
        channels, height, width = input_shape
        input = Input(shape=(height, width, channels))
        branch = edge_model.layers[3]
        output = branch(input)
        if NORMALIZATION_ON:
            output = Lambda(lambda x: K.l2_normalize(x, axis=1))(output) 
        return Model(inputs=input, outputs=output) 

def create_dagmm_model(encoder, decoder, estimation_encoder, lambd_diag=0.005):
    x_in = Input(batch_shape=encoder.input_shape)
    zc = encoder(x_in)

    decoder.name = 'reconstruction'
    x_rec = decoder(zc)
    euclid_dist = Lambda(lambda args: K.sqrt(K.sum(K.batch_flatten(K.square(args[0] - args[1])),
                                                   axis=-1, keepdims=True) /
                                             K.sum(K.batch_flatten(K.square(args[0])),
                                                   axis=-1, keepdims=True)),
                         output_shape=(1,))([x_in, x_rec])
    cos_sim = Lambda(lambda args: K.batch_dot(K.l2_normalize(K.batch_flatten(args[0]), axis=-1),
                                              K.l2_normalize(K.batch_flatten(args[1]), axis=-1),
                                              axes=-1),
                     output_shape=(1,))([x_in, x_rec])

    zr = concatenate([euclid_dist, cos_sim])
    z = concatenate([zc, zr])

    gamma = estimation_encoder(z)

    gamma_ks = [Lambda(lambda g: g[:, k:k + 1], output_shape=(1,))(gamma)
                for k in range(estimation_encoder.output_shape[-1])]

    components = [GaussianMixtureComponent(lambd_diag)([z, gamma_k])
                  for gamma_k in gamma_ks]
    density = add(components) if len(components) > 1 else components[0]
    energy = Lambda(lambda dens: -K.log(dens), name='energy')(density)

    dagmm = Model(x_in, [x_rec, energy])

    return dagmm 

def _optimize_input(self, model, layer_name, node_index, input_data, lr, steps):
        model_input = model.layers[0].input
        loss = K.max(model.get_layer(layer_name).output[...,node_index])
        grads = K.gradients(loss, model_input)[0]
        grads = K.l2_normalize(grads, axis = 1)
        iterate = K.function([model_input, K.learning_phase()], [loss, grads])
        for _ in range(steps):
            loss_value, grads_value = iterate([input_data, 0])
            input_data += grads_value * lr
        return input_data[0], loss_value > 2 

def l2Norm(x):
    return  K.l2_normalize(x, axis=-1) 

def call(self, x, mask=None):
        output = K.l2_normalize(x, self.axis)
        return output * self.gamma