Python lasagne.nonlinearities() Examples

def conv(network, batch_norm, num_layers, num_filters, filter_size, pad,
         pool_size, dropout):
    for k in range(num_layers):
        network = lnn.layers.Conv2DLayer(
            network, num_filters=num_filters,
            filter_size=filter_size,
            W=lnn.init.Orthogonal(gain=np.sqrt(2 / (1 + .1 ** 2))),
            pad=pad,
            nonlinearity=lnn.nonlinearities.rectify,
            name='Conv_{}'.format(k))
        if batch_norm:
            network = lnn.layers.batch_norm(network)

    if pool_size:
        network = lnn.layers.MaxPool2DLayer(network, pool_size=pool_size,
                                            name='Pool')
    if dropout > 0.0:
        network = lnn.layers.DropoutLayer(network, p=dropout)

    return network 

def gap(network, out_size, batch_norm,
        gap_nonlinearity, out_nonlinearity):

    gap_nonlinearity = getattr(lnn.nonlinearities, gap_nonlinearity)
    out_nonlinearity = getattr(lnn.nonlinearities, out_nonlinearity)

    # output classification layer
    network = lnn.layers.Conv2DLayer(
        network, num_filters=out_size, filter_size=1,
        nonlinearity=gap_nonlinearity, name='Output_Conv')
    if batch_norm:
        network = lnn.layers.batch_norm(network)

    network = lnn.layers.Pool2DLayer(
        network, pool_size=network.output_shape[-2:], ignore_border=False,
        mode='average_exc_pad', name='GlobalAveragePool')
    network = lnn.layers.FlattenLayer(network, name='Flatten')

    network = lnn.layers.NonlinearityLayer(
        network, nonlinearity=out_nonlinearity, name='output')

    return network 

def dense(network, batch_norm, nonlinearity, num_layers, num_units,
          dropout):

    nl = getattr(lnn.nonlinearities, nonlinearity)

    for i in range(num_layers):
        network = lnn.layers.DenseLayer(
            network, num_units=num_units, nonlinearity=nl,
            name='fc-{}'.format(i)
        )
        if batch_norm:
            network = lnn.layers.batch_norm(network)
        if dropout > 0.0:
            network = lnn.layers.DropoutLayer(network, p=dropout)

    return network 

def build_critic(input_var=None):
    from lasagne.layers import (InputLayer, Conv2DLayer, ReshapeLayer,
                                DenseLayer)
    try:
        from lasagne.layers.dnn import batch_norm_dnn as batch_norm
    except ImportError:
        from lasagne.layers import batch_norm
    from lasagne.nonlinearities import LeakyRectify
    lrelu = LeakyRectify(0.2)
    # input: (None, 1, 28, 28)
    layer = InputLayer(shape=(None, 1, 28, 28), input_var=input_var)
    # two convolutions
    layer = batch_norm(Conv2DLayer(layer, 64, 5, stride=2, pad='same',
                                   nonlinearity=lrelu))
    layer = batch_norm(Conv2DLayer(layer, 128, 5, stride=2, pad='same',
                                   nonlinearity=lrelu))
    # fully-connected layer
    layer = batch_norm(DenseLayer(layer, 1024, nonlinearity=lrelu))
    # output layer (linear and without bias)
    layer = DenseLayer(layer, 1, nonlinearity=None, b=None)
    print ("critic output:", layer.output_shape)
    return layer 

def build_BiRNN_CNN(incoming1, incoming2, num_units, mask=None, grad_clipping=0, nonlinearity=nonlinearities.tanh,
                    precompute_input=True, num_filters=20, dropout=True, in_to_out=False):
    # first get some necessary dimensions or parameters
    conv_window = 3
    _, sent_length, _ = incoming2.output_shape

    # dropout before cnn?
    if dropout:
        incoming1 = lasagne.layers.DropoutLayer(incoming1, p=0.5)

    # construct convolution layer
    cnn_layer = lasagne.layers.Conv1DLayer(incoming1, num_filters=num_filters, filter_size=conv_window, pad='full',
                                           nonlinearity=lasagne.nonlinearities.tanh, name='cnn')
    # infer the pool size for pooling (pool size should go through all time step of cnn)
    _, _, pool_size = cnn_layer.output_shape
    # construct max pool layer
    pool_layer = lasagne.layers.MaxPool1DLayer(cnn_layer, pool_size=pool_size)
    # reshape the layer to match rnn incoming layer [batch * sent_length, num_filters, 1] --> [batch, sent_length, num_filters]
    output_cnn_layer = lasagne.layers.reshape(pool_layer, (-1, sent_length, [1]))

    # finally, concatenate the two incoming layers together.
    incoming = lasagne.layers.concat([output_cnn_layer, incoming2], axis=2)

    return build_BiRNN(incoming, num_units, mask=mask, grad_clipping=grad_clipping, nonlinearity=nonlinearity,
                       precompute_input=precompute_input, dropout=dropout, in_to_out=in_to_out) 

def build_BiLSTM_CNN(incoming1, incoming2, num_units, mask=None, grad_clipping=0, precompute_input=True,
                     peepholes=False, num_filters=20, dropout=True, in_to_out=False):
    # first get some necessary dimensions or parameters
    conv_window = 3
    _, sent_length, _ = incoming2.output_shape

    # dropout before cnn?
    if dropout:
        incoming1 = lasagne.layers.DropoutLayer(incoming1, p=0.5)

    # construct convolution layer
    cnn_layer = lasagne.layers.Conv1DLayer(incoming1, num_filters=num_filters, filter_size=conv_window, pad='full',
                                           nonlinearity=lasagne.nonlinearities.tanh, name='cnn')
    # infer the pool size for pooling (pool size should go through all time step of cnn)
    _, _, pool_size = cnn_layer.output_shape
    # construct max pool layer
    pool_layer = lasagne.layers.MaxPool1DLayer(cnn_layer, pool_size=pool_size)
    # reshape the layer to match lstm incoming layer [batch * sent_length, num_filters, 1] --> [batch, sent_length, num_filters]
    output_cnn_layer = lasagne.layers.reshape(pool_layer, (-1, sent_length, [1]))

    # finally, concatenate the two incoming layers together.
    incoming = lasagne.layers.concat([output_cnn_layer, incoming2], axis=2)

    return build_BiLSTM(incoming, num_units, mask=mask, grad_clipping=grad_clipping, peepholes=peepholes,
                        precompute_input=precompute_input, dropout=dropout, in_to_out=in_to_out) 

def build_critic(input_var=None):
    from lasagne.layers import (InputLayer, Conv2DLayer, ReshapeLayer,
                                DenseLayer)
    try:
        from lasagne.layers.dnn import batch_norm_dnn as batch_norm
    except ImportError:
        from lasagne.layers import batch_norm
    from lasagne.nonlinearities import LeakyRectify
    lrelu = LeakyRectify(0.2)
    # input: (None, 1, 28, 28)
    layer = InputLayer(shape=(None, 1, 28, 28), input_var=input_var)
    # two convolutions
    layer = batch_norm(Conv2DLayer(layer, 64, 5, stride=2, pad='same',
                                   nonlinearity=lrelu))
    layer = batch_norm(Conv2DLayer(layer, 128, 5, stride=2, pad='same',
                                   nonlinearity=lrelu))
    # fully-connected layer
    layer = batch_norm(DenseLayer(layer, 1024, nonlinearity=lrelu))
    # output layer (linear)
    layer = DenseLayer(layer, 1, nonlinearity=None)
    print ("critic output:", layer.output_shape)
    return layer 

def build_classfication_model_from_vgg16 ():
    layer_list, vgg_whole, vgg_input_var = vgg16.vgg16.build_model();

    vgg_cut = layer_list['fc8'];

    aug_var = theano.tensor.matrix('aug_var');
    aug_layer = lasagne.layers.InputLayer(shape=(None, aug_dim), input_var = aug_var);

    layer_list['aggregate_layer'] = lasagne.layers.ConcatLayer([vgg_cut,aug_layer], axis = 1);

    layer_list['last_sigmoid'] = lasagne.layers.DenseLayer(incoming=layer_list['aggregate_layer'], num_units=n_binaryclassifier, nonlinearity=lasagne.nonlinearities.sigmoid);
    network = layer_list['last_sigmoid'];

    latter_param = [layer_list['last_sigmoid'].W, layer_list['last_sigmoid'].b];
    all_param = lasagne.layers.get_all_params(network, trainable=True);

    return network, vgg_whole, layer_list, all_param, latter_param, vgg_input_var, aug_var; 

def build_classfication_model_from_vgg16 ():
    layer_list, vgg_whole, vgg_input_var = vgg16.vgg16.build_model();

    vgg_cut = layer_list['fc8'];

    aug_var = theano.tensor.matrix('aug_var');
    aug_layer = lasagne.layers.InputLayer(shape=(None, aug_dim), input_var = aug_var);

    layer_list['aggregate_layer'] = lasagne.layers.ConcatLayer([vgg_cut,aug_layer], axis = 1);

    layer_list['last_sigmoid'] = lasagne.layers.DenseLayer(incoming=layer_list['aggregate_layer'], num_units=n_binaryclassifier, nonlinearity=lasagne.nonlinearities.sigmoid);
    network = layer_list['last_sigmoid'];

    latter_param = [layer_list['last_sigmoid'].W, layer_list['last_sigmoid'].b];
    all_param = lasagne.layers.get_all_params(network, trainable=True);

    return network, vgg_whole, layer_list, all_param, latter_param, vgg_input_var, aug_var; 

def build_classfication_model_from_vgg16 ():
    layer_list, vgg_whole, vgg_input_var = vgg16.vgg16.build_model();

    vgg_cut = layer_list['fc8'];

    aug_var = theano.tensor.matrix('aug_var');
    aug_layer = lasagne.layers.InputLayer(shape=(None, aug_dim), input_var = aug_var);

    layer_list['aggregate_layer'] = lasagne.layers.ConcatLayer([vgg_cut,aug_layer], axis = 1);

    layer_list['last_sigmoid'] = lasagne.layers.DenseLayer(incoming=layer_list['aggregate_layer'], num_units=n_binaryclassifier, nonlinearity=lasagne.nonlinearities.sigmoid);
    network = layer_list['last_sigmoid'];

    latter_param = [layer_list['last_sigmoid'].W, layer_list['last_sigmoid'].b];
    all_param = lasagne.layers.get_all_params(network, trainable=True);

    return network, vgg_whole, layer_list, all_param, latter_param, vgg_input_var, aug_var; 

def build_classfication_model_from_vgg16 ():
    layer_list, vgg_whole, vgg_input_var = vgg16.vgg16.build_model();

    vgg_cut = layer_list['fc8'];

    aug_var = theano.tensor.matrix('aug_var');
    aug_layer = lasagne.layers.InputLayer(shape=(None, aug_dim), input_var = aug_var);

    layer_list['aggregate_layer'] = lasagne.layers.ConcatLayer([vgg_cut,aug_layer], axis = 1);

    layer_list['last_sigmoid'] = lasagne.layers.DenseLayer(incoming=layer_list['aggregate_layer'], num_units=n_binaryclassifier, nonlinearity=lasagne.nonlinearities.sigmoid);
    network = layer_list['last_sigmoid'];

    latter_param = [layer_list['last_sigmoid'].W, layer_list['last_sigmoid'].b];
    all_param = lasagne.layers.get_all_params(network, trainable=True);

    return network, vgg_whole, layer_list, all_param, latter_param, vgg_input_var, aug_var; 

def build_generator(input_var=None):
    from lasagne.layers import InputLayer, ReshapeLayer, DenseLayer
    try:
        from lasagne.layers import TransposedConv2DLayer as Deconv2DLayer
    except ImportError:
        raise ImportError("Your Lasagne is too old. Try the bleeding-edge "
                          "version: http://lasagne.readthedocs.io/en/latest/"
                          "user/installation.html#bleeding-edge-version")
    try:
        from lasagne.layers.dnn import batch_norm_dnn as batch_norm
    except ImportError:
        from lasagne.layers import batch_norm
    from lasagne.nonlinearities import sigmoid
    # input: 100dim
    layer = InputLayer(shape=(None, 100), input_var=input_var)
    # fully-connected layer
    layer = batch_norm(DenseLayer(layer, 1024))
    # project and reshape
    layer = batch_norm(DenseLayer(layer, 128*7*7))
    layer = ReshapeLayer(layer, ([0], 128, 7, 7))
    # two fractional-stride convolutions
    layer = batch_norm(Deconv2DLayer(layer, 64, 5, stride=2, crop='same',
                                     output_size=14))
    layer = Deconv2DLayer(layer, 1, 5, stride=2, crop='same', output_size=28,
                          nonlinearity=sigmoid)
    print ("Generator output:", layer.output_shape)
    return layer 

def recurrent(network, mask_in, num_rec_units, num_layers, dropout,
              bidirectional, nonlinearity):

    if nonlinearity != 'LSTM':
        nl = getattr(lnn.nonlinearities, nonlinearity)

        def add_layer(prev_layer, **kwargs):
            return lnn.layers.RecurrentLayer(
                prev_layer, num_units=num_rec_units, mask_input=mask_in,
                nonlinearity=nl,
                W_in_to_hid=lnn.init.GlorotUniform(),
                W_hid_to_hid=lnn.init.Orthogonal(gain=np.sqrt(2) / 2),
                **kwargs)

    else:
        def add_layer(prev_layer, **kwargs):
            return lnn.layers.LSTMLayer(
                prev_layer, num_units=num_rec_units, mask_input=mask_in,
                **kwargs
            )

    fwd = network
    for i in range(num_layers):
        fwd = add_layer(fwd, name='rec_fwd_{}'.format(i))
        if dropout > 0.:
            fwd = lnn.layers.DropoutLayer(fwd, p=dropout)

    if not bidirectional:
        return network

    bck = network
    for i in range(num_layers):
        bck = add_layer(bck, name='rec_bck_{}'.format(i), backwards=True)
        if dropout > 0:
            bck = lnn.layers.DropoutLayer(bck, p=dropout)

    # combine the forward and backward recurrent layers...
    network = lnn.layers.ConcatLayer([fwd, bck], name='fwd + bck', axis=-1)
    return network 

def __init__(self, incoming, num_filters, filter_size, stride=(1, 1),
                 crop=0, untie_biases=False,
                 W=initmethod(), b=lasagne.init.Constant(0.),
                 nonlinearity=lasagne.nonlinearities.rectify, flip_filters=False,
                 **kwargs):
        super(DeconvLayer, self).__init__(
                incoming, num_filters, filter_size, stride, crop, untie_biases,
                W, b, nonlinearity, flip_filters, n=2, **kwargs)
        # rename self.crop to self.pad
        self.crop = self.pad
        del self.pad 

def build_generator(input_var=None):
    from lasagne.layers import InputLayer, ReshapeLayer, DenseLayer
    try:
        from lasagne.layers import TransposedConv2DLayer as Deconv2DLayer
    except ImportError:
        raise ImportError("Your Lasagne is too old. Try the bleeding-edge "
                          "version: http://lasagne.readthedocs.io/en/latest/"
                          "user/installation.html#bleeding-edge-version")
    try:
        from lasagne.layers.dnn import batch_norm_dnn as batch_norm
    except ImportError:
        from lasagne.layers import batch_norm
    from lasagne.nonlinearities import sigmoid
    # input: 100dim
    layer = InputLayer(shape=(None, 100), input_var=input_var)
    # fully-connected layer
    layer = batch_norm(DenseLayer(layer, 1024))
    # project and reshape
    layer = batch_norm(DenseLayer(layer, 128*7*7))
    layer = ReshapeLayer(layer, ([0], 128, 7, 7))
    # two fractional-stride convolutions
    layer = batch_norm(Deconv2DLayer(layer, 64, 5, stride=2, crop='same',
                                     output_size=14))
    layer = Deconv2DLayer(layer, 1, 5, stride=2, crop='same', output_size=28,
                          nonlinearity=sigmoid)
    print ("Generator output:", layer.output_shape)
    return layer 

def build_generator(input_var=None, verbose=False):
    from lasagne.layers import InputLayer, ReshapeLayer, DenseLayer
    try:
        from lasagne.layers import TransposedConv2DLayer as Deconv2DLayer
    except ImportError:
        raise ImportError("Your Lasagne is too old. Try the bleeding-edge "
                          "version: http://lasagne.readthedocs.io/en/latest/"
                          "user/installation.html#bleeding-edge-version")
    try:
        from lasagne.layers.dnn import batch_norm_dnn as batch_norm
    except ImportError:
        from lasagne.layers import batch_norm
    from lasagne.nonlinearities import sigmoid
    # input: 100dim
    layer = InputLayer(shape=(None, 100), input_var=input_var)
    # # fully-connected layer
    # layer = batch_norm(DenseLayer(layer, 1024))
    # project and reshape
    layer = batch_norm(DenseLayer(layer, 1024*4*4))
    layer = ReshapeLayer(layer, ([0], 1024, 4, 4))
    # two fractional-stride convolutions
    layer = batch_norm(Deconv2DLayer(layer, 512, 5, stride=2, crop='same',
                                     output_size=8))
    layer = batch_norm(Deconv2DLayer(layer, 256, 5, stride=2, crop='same',
                                     output_size=16))
    layer = Deconv2DLayer(layer, 3, 5, stride=2, crop='same', output_size=32,
                          nonlinearity=sigmoid)
    if verbose: print ("Generator output:", layer.output_shape)
    return layer 

def __init__(self):
        self.num_layers = 1  # Number of hidden layers
        self.num_nodes = 10  # Number of nodes per hidden layer
        self.num_inputs = None  # Number of scalar invariants
        self.num_tensor_basis = None  # Number of tensors in the tensor basis
        self.nonlinearity = "LeakyRectify" # non-linearity string conforming to lasagne.nonlinearities tags
        self.nonlinearity_keywords = {}
        self.nonlinearity_keywords["leakiness"] = "0.1" # Leakiness of leaky ReLU activation functions 

def _build_NN(self):
        """
        Builds a TBNN with the number of hidden layers and hidden nodes specified in self.structure
        Right now, it is hard coded to use a Leaky ReLU activation function for all hidden layers
        """
        # determine type of non-linearity first
        nonlinearity_string = list("lasagne.nonlinearities."+self.structure.nonlinearity)
        # check if upper to know if there are args and () or not
        if self.structure.nonlinearity[0].isupper():
            nonlinearity_string += list("(")
            # add keyword options
            for key in self.structure.nonlinearity_keywords:
                nonlinearity_string += list(key+"="+self.structure.nonlinearity_keywords[key]+",")
            if self.structure.nonlinearity_keywords:
                nonlinearity_string[-1] = ")"
            else:
                nonlinearity_string += list(")")
        nonlinearity = eval("".join(nonlinearity_string))
        
        input_x = T.dmatrix('input_x')
        input_tb = T.dtensor3('input_tb')
        input_layer = lasagne.layers.InputLayer(shape=(None, self.structure.num_inputs), input_var=input_x)
        hidden_layer = lasagne.layers.DenseLayer(input_layer, num_units=self.structure.num_nodes,
            nonlinearity=nonlinearity, W=lasagne.init.HeUniform(gain=np.sqrt(2.0)))
        for i in xrange(self.structure.num_layers - 1):
            hidden_layer = lasagne.layers.DenseLayer(hidden_layer, num_units=self.structure.num_nodes,
                nonlinearity=nonlinearity, W=lasagne.init.HeUniform(gain=np.sqrt(2.0)))
        linear_layer = lasagne.layers.DenseLayer(hidden_layer, num_units=self.structure.num_tensor_basis,
            nonlinearity=None, W=lasagne.init.HeUniform(gain=np.sqrt(2.0)))

        tensor_layer = lasagne.layers.InputLayer(shape=(None, self.structure.num_tensor_basis, 9), input_var=input_tb)
        merge_layer = TensorLayer([linear_layer, tensor_layer])
        self.network = merge_layer 

def build_net(in_shape, out_size, model):
    # input variables
    input_var = (tt.tensor3('input', dtype='float32')
                 if len(in_shape) > 1 else
                 tt.matrix('input', dtype='float32'))
    target_var = tt.matrix('target_output', dtype='float32')

    # stack more layers
    network = lnn.layers.InputLayer(
        name='input', shape=(None,) + in_shape, input_var=input_var)

    if 'conv' in model and model['conv']:
        # reshape to 1 "color" channel
        network = lnn.layers.reshape(
            network, shape=(-1, 1) + in_shape, name='reshape')

        for c in sorted(model['conv'].keys()):
            network = blocks.conv(network, **model['conv'][c])

    # no more output layer if gap is already there!
    if 'gap' in model and model['gap']:
        network = blocks.gap(network, out_size=out_size,
                             out_nonlinearity=model['out_nonlinearity'],
                             **model['gap'])
    else:
        if 'dense' in model and model['dense']:
            network = blocks.dense(network, **model['dense'])

        # output layer
        out_nl = getattr(lnn.nonlinearities, model['out_nonlinearity'])
        network = lnn.layers.DenseLayer(
            network, name='output', num_units=out_size,
            nonlinearity=out_nl)

    return network, input_var, target_var 

def get_output_for(self,input, **kwargs):
        if input.ndim > 2:
            input = input.flatten(2)
        
        activation = T.dot(input, self.W*self.weights_mask)            
        
        if self.b is not None:
            activation = activation + self.b.dimshuffle('x', 0)
        return self.nonlinearity(activation)        

# Conditioning Masked Layer 
# Currently not used.       
# class CML(MaskedLayer):

    # def __init__(self, incoming, num_units, mask_generator,use_cond_mask=False,U=lasagne.init.GlorotUniform(),W=lasagne.init.GlorotUniform(),
                 # b=init.Constant(0.), nonlinearity=lasagne.nonlinearities.rectify, **kwargs):
        # super(CML, self).__init__(incoming, num_units, mask_generator,W,
                 # b, nonlinearity,**kwargs)
        
        # self.use_cond_mask=use_cond_mask
        # if use_cond_mask:            
            # self.U = self.add_param(spec = U,
                                    # shape = (num_inputs, num_units),
                                    # name='U',
                                    # trainable=True,
                                    # regularizable=False)theano.shared(value=self.weights_initialization((self.n_in, self.n_out)), name=self.name+'U', borrow=True)
            # self.add_param(self.U,name = 
    # def get_output_for(self,input,**kwargs):
       # lin = self.lin_output = T.dot(input, self.W * self.weights_mask) + self.b  
       # if self.use_cond_mask:
           # lin = lin+T.dot(T.ones_like(input), self.U * self.weights_mask)
       # return lin if self._activation is None else self._activation(lin) 


       
# Made layer, adopted from M.Germain 

def __init__(self, incoming, num_units, mask_generator,layerIdx,W=lasagne.init.GlorotUniform(),
                 b=lasagne.init.Constant(0.), nonlinearity=lasagne.nonlinearities.rectify, **kwargs):
        super(MaskedLayer, self).__init__(incoming, num_units, W,b, nonlinearity,**kwargs)
        self.mask_generator = mask_generator
        num_inputs = int(np.prod(self.input_shape[1:]))
        self.weights_mask = self.add_param(spec = np.ones((num_inputs, num_units),dtype=np.float32),
                                           shape = (num_inputs, num_units),
                                           name='weights_mask',
                                           trainable=False,
                                           regularizable=False)
        self.layerIdx = layerIdx
        self.shuffle_update = [(self.weights_mask, mask_generator.get_mask_layer_UPDATE(self.layerIdx))] 

def build_BiLSTM_HighCNN(incoming1, incoming2, num_units, mask=None, grad_clipping=0, precompute_input=True,
                         peepholes=False, num_filters=20, dropout=True, in_to_out=False):
    # first get some necessary dimensions or parameters
    conv_window = 3
    _, sent_length, _ = incoming2.output_shape

    # dropout before cnn
    if dropout:
        incoming1 = lasagne.layers.DropoutLayer(incoming1, p=0.5)

    # construct convolution layer
    cnn_layer = lasagne.layers.Conv1DLayer(incoming1, num_filters=num_filters, filter_size=conv_window, pad='full',
                                           nonlinearity=lasagne.nonlinearities.tanh, name='cnn')
    # infer the pool size for pooling (pool size should go through all time step of cnn)
    _, _, pool_size = cnn_layer.output_shape
    # construct max pool layer
    pool_layer = lasagne.layers.MaxPool1DLayer(cnn_layer, pool_size=pool_size)
    # reshape the layer to match highway incoming layer [batch * sent_length, num_filters, 1] --> [batch * sent_length, num_filters]
    output_cnn_layer = lasagne.layers.reshape(pool_layer, ([0], -1))

    # dropout after cnn?
    # if dropout:
    # output_cnn_layer = lasagne.layers.DropoutLayer(output_cnn_layer, p=0.5)

    # construct highway layer
    highway_layer = HighwayDenseLayer(output_cnn_layer, nonlinearity=nonlinearities.rectify)

    # reshape the layer to match lstm incoming layer [batch * sent_length, num_filters] --> [batch, sent_length, number_filters]
    output_highway_layer = lasagne.layers.reshape(highway_layer, (-1, sent_length, [1]))

    # finally, concatenate the two incoming layers together.
    incoming = lasagne.layers.concat([output_highway_layer, incoming2], axis=2)

    return build_BiLSTM(incoming, num_units, mask=mask, grad_clipping=grad_clipping, peepholes=peepholes,
                        precompute_input=precompute_input, dropout=dropout, in_to_out=in_to_out) 

def build_BiRNN(incoming, num_units, mask=None, grad_clipping=0, nonlinearity=nonlinearities.tanh,
                precompute_input=True, dropout=True, in_to_out=False):
    # construct the forward and backward rnns. Now, Ws are initialized by He initializer with default arguments.
    # Need to try other initializers for specific tasks.

    # dropout for incoming
    if dropout:
        incoming = lasagne.layers.DropoutLayer(incoming, p=0.5)

    rnn_forward = lasagne.layers.RecurrentLayer(incoming, num_units,
                                                mask_input=mask, grad_clipping=grad_clipping,
                                                nonlinearity=nonlinearity, precompute_input=precompute_input,
                                                W_in_to_hid=lasagne.init.GlorotUniform(),
                                                W_hid_to_hid=lasagne.init.GlorotUniform(), name='forward')
    rnn_backward = lasagne.layers.RecurrentLayer(incoming, num_units,
                                                 mask_input=mask, grad_clipping=grad_clipping,
                                                 nonlinearity=nonlinearity, precompute_input=precompute_input,
                                                 W_in_to_hid=lasagne.init.GlorotUniform(),
                                                 W_hid_to_hid=lasagne.init.GlorotUniform(), backwards=True,
                                                 name='backward')

    # concatenate the outputs of forward and backward RNNs to combine them.
    concat = lasagne.layers.concat([rnn_forward, rnn_backward], axis=2, name="bi-rnn")

    # dropout for output
    if dropout:
        concat = lasagne.layers.DropoutLayer(concat, p=0.5)

    if in_to_out:
        concat = lasagne.layers.concat([concat, incoming], axis=2)

    # the shape of BiRNN output (concat) is (batch_size, input_length, 2 * num_hidden_units)
    return concat 

def __init__(self, W_in=init.GlorotUniform(), W_hid=init.GlorotUniform(),
                 W_cell=init.GlorotUniform(), b=init.Constant(0.),
                 nonlinearity=nonlinearities.sigmoid):
        self.W_in = W_in
        self.W_hid = W_hid
        # Don't store a cell weight vector when cell is None
        if W_cell is not None:
            self.W_cell = W_cell
        self.b = b
        # For the nonlinearity, if None is supplied, use identity
        if nonlinearity is None:
            self.nonlinearity = nonlinearities.identity
        else:
            self.nonlinearity = nonlinearity 

def build_model():
    '''
    Builds C3D model

    Returns
    -------
    dict
        A dictionary containing the network layers, where the output layer is at key 'prob'
    '''
    net = {}
    net['input'] = InputLayer((None, 3, 16, 112, 112))

    # ----------- 1st layer group ---------------
    net['conv1a'] = Conv3DDNNLayer(net['input'], 64, (3,3,3), pad=1,nonlinearity=lasagne.nonlinearities.rectify,flip_filters=False)
    net['pool1']  = MaxPool3DDNNLayer(net['conv1a'],pool_size=(1,2,2),stride=(1,2,2))

    # ------------- 2nd layer group --------------
    net['conv2a'] = Conv3DDNNLayer(net['pool1'], 128, (3,3,3), pad=1,nonlinearity=lasagne.nonlinearities.rectify)
    net['pool2']  = MaxPool3DDNNLayer(net['conv2a'],pool_size=(2,2,2),stride=(2,2,2))

    # ----------------- 3rd layer group --------------
    net['conv3a'] = Conv3DDNNLayer(net['pool2'], 256, (3,3,3), pad=1,nonlinearity=lasagne.nonlinearities.rectify)
    net['conv3b'] = Conv3DDNNLayer(net['conv3a'], 256, (3,3,3), pad=1,nonlinearity=lasagne.nonlinearities.rectify)
    net['pool3']  = MaxPool3DDNNLayer(net['conv3b'],pool_size=(2,2,2),stride=(2,2,2))

    # ----------------- 4th layer group --------------
    net['conv4a'] = Conv3DDNNLayer(net['pool3'], 512, (3,3,3), pad=1,nonlinearity=lasagne.nonlinearities.rectify)
    net['conv4b'] = Conv3DDNNLayer(net['conv4a'], 512, (3,3,3), pad=1,nonlinearity=lasagne.nonlinearities.rectify)
    net['pool4']  = MaxPool3DDNNLayer(net['conv4b'],pool_size=(2,2,2),stride=(2,2,2))

    # ----------------- 5th layer group --------------
    net['conv5a'] = Conv3DDNNLayer(net['pool4'], 512, (3,3,3), pad=1,nonlinearity=lasagne.nonlinearities.rectify)
    net['conv5b'] = Conv3DDNNLayer(net['conv5a'], 512, (3,3,3), pad=1,nonlinearity=lasagne.nonlinearities.rectify)
    # We need a padding layer, as C3D only pads on the right, which cannot be done with a theano pooling layer
    net['pad']    = PadLayer(net['conv5b'],width=[(0,1),(0,1)], batch_ndim=3)
    net['pool5']  = MaxPool3DDNNLayer(net['pad'],pool_size=(2,2,2),pad=(0,0,0),stride=(2,2,2))
    net['fc6-1']  = DenseLayer(net['pool5'], num_units=4096,nonlinearity=lasagne.nonlinearities.rectify)
    net['fc7-1']  = DenseLayer(net['fc6-1'], num_units=4096,nonlinearity=lasagne.nonlinearities.rectify)
    net['fc8-1']  = DenseLayer(net['fc7-1'], num_units=487, nonlinearity=None)
    net['prob']  = NonlinearityLayer(net['fc8-1'], softmax)

    return net 

def network(self):
        if self._network is not None:
            return self._network

        # Build the computational graph using a dummy input.
        import lasagne
        from lasagne.layers.dnn import Conv2DDNNLayer as ConvLayer
        from lasagne.layers import ElemwiseSumLayer, NonlinearityLayer, ExpressionLayer, PadLayer, InputLayer, FlattenLayer, SliceLayer
        # from lasagne.layers import batch_norm
        from lasagne.nonlinearities import rectify

        self._network_in = InputLayer(shape=(None, self.nb_channels,) + self.image_shape, input_var=None)

        convnet_layers = [self._network_in]
        convnet_layers_preact = [self._network_in]
        layer_blueprints = list(map(str.strip, self.convnet_blueprint.split("->")))
        for i, layer_blueprint in enumerate(layer_blueprints, start=1):
            "64@3x3(valid) -> 64@3x3(full)"
            nb_filters, rest = layer_blueprint.split("@")
            filter_shape, rest = rest.split("(")
            nb_filters = int(nb_filters)
            filter_shape = tuple(map(int, filter_shape.split("x")))
            pad = rest[:-1]

            preact = ConvLayer(convnet_layers[-1], num_filters=nb_filters, filter_size=filter_shape, stride=(1, 1), nonlinearity=None, pad=pad, W=lasagne.init.HeNormal(gain='relu'))

            if i > len(layer_blueprints) // 2 and i != len(layer_blueprints):
                shortcut = convnet_layers_preact[len(layer_blueprints)-i]
                if i == len(layer_blueprints):
                    if preact.output_shape[1] != shortcut.output_shape[1]:
                        shortcut = SliceLayer(shortcut, slice(0, 1), axis=1)
                    else:
                        raise NameError("Something is wrong.")

                print("Shortcut from {} to {}".format(len(layer_blueprints)-i, i))
                preact = ElemwiseSumLayer([preact, shortcut])

            convnet_layers_preact.append(preact)

            layer = NonlinearityLayer(preact, nonlinearity=rectify)
            convnet_layers.append(layer)

        self._network = FlattenLayer(preact)
        # network = DenseLayer(l, num_units=int(np.prod(self.image_shape)),
        #                      W=lasagne.init.HeNormal(),
        #                      nonlinearity=None)

        print("Nb. of parameters in model: {}".format(lasagne.layers.count_params(self._network, trainable=True)))
        return self._network 

def network(self):
        if self._network is not None:
            return self._network

        # Build the computational graph using a dummy input.
        import lasagne
        from lasagne.layers.dnn import Conv2DDNNLayer as ConvLayer
        from lasagne.layers import ElemwiseSumLayer, NonlinearityLayer, InputLayer, FlattenLayer, DenseLayer
        from lasagne.layers import batch_norm
        from lasagne.nonlinearities import rectify

        self._network_in = InputLayer(shape=(None, self.nb_channels,) + self.image_shape, input_var=None)
        network_out = []

        if self.convnet_blueprint is not None:
            convnet_layers = [self._network_in]
            layer_blueprints = list(map(str.strip, self.convnet_blueprint.split("->")))
            for i, layer_blueprint in enumerate(layer_blueprints, start=1):
                # eg. "64@3x3(valid) -> 64@3x3(full)"
                nb_filters, rest = layer_blueprint.split("@")
                filter_shape, rest = rest.split("(")
                nb_filters = int(nb_filters)
                filter_shape = tuple(map(int, filter_shape.split("x")))
                pad = rest[:-1]

                preact = ConvLayer(convnet_layers[-1], num_filters=nb_filters, filter_size=filter_shape, stride=(1, 1),
                                   nonlinearity=None, pad=pad, W=lasagne.init.HeNormal(gain='relu'),
                                   name="layer_{}_conv".format(i))

                if self.use_batch_norm:
                    preact = batch_norm(preact)

                layer = NonlinearityLayer(preact, nonlinearity=rectify)
                convnet_layers.append(layer)

            network_out.append(FlattenLayer(preact))

        if self.fullnet_blueprint is not None:
            fullnet_layers = [FlattenLayer(self._network_in)]
            layer_blueprints = list(map(str.strip, self.fullnet_blueprint.split("->")))
            for i, layer_blueprint in enumerate(layer_blueprints, start=1):
                # e.g. "500 -> 500 -> 784"
                hidden_size = int(layer_blueprint)

                preact = DenseLayer(fullnet_layers[-1], num_units=hidden_size,
                                    nonlinearity=None, W=lasagne.init.HeNormal(gain='relu'),
                                    name="layer_{}_dense".format(i))

                if self.use_batch_norm:
                    preact = batch_norm(preact)

                layer = NonlinearityLayer(preact, nonlinearity=rectify)
                fullnet_layers.append(layer)

            network_out.append(preact)

        self._network = ElemwiseSumLayer(network_out)
        # TODO: sigmoid should be applied here instead of within loss function.
        print("Nb. of parameters in model: {}".format(lasagne.layers.count_params(self._network, trainable=True)))
        return self._network 

def build_BiGRU(incoming, num_units, mask=None, grad_clipping=0, precompute_input=True, dropout=True, in_to_out=False):
    # construct the forward and backward grus. Now, Ws are initialized by Glorot initializer with default arguments.
    # Need to try other initializers for specific tasks.

    # dropout for incoming
    if dropout:
        incoming = lasagne.layers.DropoutLayer(incoming, p=0.5)

    # according to Jozefowicz et al.(2015), init bias of forget gate to 1.
    resetgate_forward = Gate(W_in=lasagne.init.GlorotUniform(), W_hid=lasagne.init.GlorotUniform(),
                             W_cell=lasagne.init.Uniform(range=0.1), b=lasagne.init.Constant(1.))
    updategate_forward = Gate(W_in=lasagne.init.GlorotUniform(), W_hid=lasagne.init.GlorotUniform(),
                              W_cell=lasagne.init.Uniform(range=0.1))
    # now use tanh for nonlinear function of hidden gate
    hidden_forward = Gate(W_in=lasagne.init.GlorotUniform(), W_hid=lasagne.init.GlorotUniform(), W_cell=None,
                          nonlinearity=nonlinearities.tanh)
    gru_forward = lasagne.layers.GRULayer(incoming, num_units, mask_input=mask, grad_clipping=grad_clipping,
                                          precompute_input=precompute_input,
                                          resetgate=resetgate_forward, updategate=updategate_forward,
                                          hidden_update=hidden_forward, name='forward')

    # according to Jozefowicz et al.(2015), init bias of forget gate to 1.
    resetgate_backward = Gate(W_in=lasagne.init.GlorotUniform(), W_hid=lasagne.init.GlorotUniform(),
                              W_cell=lasagne.init.Uniform(range=0.1), b=lasagne.init.Constant(1.))
    updategate_backward = Gate(W_in=lasagne.init.GlorotUniform(), W_hid=lasagne.init.GlorotUniform(),
                               W_cell=lasagne.init.Uniform(range=0.1))
    # now use tanh for nonlinear function of hidden gate
    hidden_backward = Gate(W_in=lasagne.init.GlorotUniform(), W_hid=lasagne.init.GlorotUniform(), W_cell=None,
                           nonlinearity=nonlinearities.tanh)
    gru_backward = lasagne.layers.GRULayer(incoming, num_units, mask_input=mask, grad_clipping=grad_clipping,
                                           precompute_input=precompute_input, backwards=True,
                                           resetgate=resetgate_backward, updategate=updategate_backward,
                                           hidden_update=hidden_backward, name='backward')

    # concatenate the outputs of forward and backward GRUs to combine them.
    concat = lasagne.layers.concat([gru_forward, gru_backward], axis=2, name="bi-gru")

    # dropout for output
    if dropout:
        concat = lasagne.layers.DropoutLayer(concat, p=0.5)

    if in_to_out:
        concat = lasagne.layers.concat([concat, incoming], axis=2)

    # the shape of BiRNN output (concat) is (batch_size, input_length, 2 * num_hidden_units)
    return concat 

def build_BiLSTM(incoming, num_units, mask=None, grad_clipping=0, precompute_input=True, peepholes=False, dropout=True,
                 in_to_out=False):
    # construct the forward and backward rnns. Now, Ws are initialized by Glorot initializer with default arguments.
    # Need to try other initializers for specific tasks.

    # dropout for incoming
    if dropout:
        incoming = lasagne.layers.DropoutLayer(incoming, p=0.5)

    ingate_forward = Gate(W_in=lasagne.init.GlorotUniform(), W_hid=lasagne.init.GlorotUniform(),
                          W_cell=lasagne.init.Uniform(range=0.1))
    outgate_forward = Gate(W_in=lasagne.init.GlorotUniform(), W_hid=lasagne.init.GlorotUniform(),
                           W_cell=lasagne.init.Uniform(range=0.1))
    # according to Jozefowicz et al.(2015), init bias of forget gate to 1.
    forgetgate_forward = Gate(W_in=lasagne.init.GlorotUniform(), W_hid=lasagne.init.GlorotUniform(),
                              W_cell=lasagne.init.Uniform(range=0.1), b=lasagne.init.Constant(1.))
    # now use tanh for nonlinear function of cell, need to try pure linear cell
    cell_forward = Gate(W_in=lasagne.init.GlorotUniform(), W_hid=lasagne.init.GlorotUniform(), W_cell=None,
                        nonlinearity=nonlinearities.tanh)
    lstm_forward = lasagne.layers.LSTMLayer(incoming, num_units, mask_input=mask, grad_clipping=grad_clipping,
                                            nonlinearity=nonlinearities.tanh, peepholes=peepholes,
                                            precompute_input=precompute_input,
                                            ingate=ingate_forward, outgate=outgate_forward,
                                            forgetgate=forgetgate_forward, cell=cell_forward, name='forward')

    ingate_backward = Gate(W_in=lasagne.init.GlorotUniform(), W_hid=lasagne.init.GlorotUniform(),
                           W_cell=lasagne.init.Uniform(range=0.1))
    outgate_backward = Gate(W_in=lasagne.init.GlorotUniform(), W_hid=lasagne.init.GlorotUniform(),
                            W_cell=lasagne.init.Uniform(range=0.1))
    # according to Jozefowicz et al.(2015), init bias of forget gate to 1.
    forgetgate_backward = Gate(W_in=lasagne.init.GlorotUniform(), W_hid=lasagne.init.GlorotUniform(),
                               W_cell=lasagne.init.Uniform(range=0.1), b=lasagne.init.Constant(1.))
    # now use tanh for nonlinear function of cell, need to try pure linear cell
    cell_backward = Gate(W_in=lasagne.init.GlorotUniform(), W_hid=lasagne.init.GlorotUniform(), W_cell=None,
                         nonlinearity=nonlinearities.tanh)
    lstm_backward = lasagne.layers.LSTMLayer(incoming, num_units, mask_input=mask, grad_clipping=grad_clipping,
                                             nonlinearity=nonlinearities.tanh, peepholes=peepholes,
                                             precompute_input=precompute_input, backwards=True,
                                             ingate=ingate_backward, outgate=outgate_backward,
                                             forgetgate=forgetgate_backward, cell=cell_backward, name='backward')

    # concatenate the outputs of forward and backward RNNs to combine them.
    concat = lasagne.layers.concat([lstm_forward, lstm_backward], axis=2, name="bi-lstm")

    # dropout for output
    if dropout:
        concat = lasagne.layers.DropoutLayer(concat, p=0.5)

    if in_to_out:
        concat = lasagne.layers.concat([concat, incoming], axis=2)

    # the shape of BiRNN output (concat) is (batch_size, input_length, 2 * num_hidden_units)
    return concat