Python theano.tensor.lmatrix() Examples

def test_blocksparse_inplace_outer_opt():
    b = tensor.fmatrix()
    W = tensor.ftensor4()
    h = tensor.ftensor3()
    iIdx = tensor.lmatrix()
    oIdx = tensor.lmatrix()

    o = sparse_block_dot(W, h, iIdx, b, oIdx)

    theano.printing.debugprint(tensor.grad(o.sum(), wrt=W))

    f = theano.function([W, h, iIdx, b, oIdx],
                        [o, tensor.grad(o.sum(), wrt=W)])
    assert hasattr(f.maker.fgraph.outputs[0].tag, 'trace')

    if theano.config.mode == "FAST_COMPILE":
        assert not f.maker.fgraph.toposort()[-1].op.inplace
    else:
        assert f.maker.fgraph.toposort()[-1].op.inplace 

def test_blocksparse_inplace_gemv_opt():
    b = tensor.fmatrix()
    W = tensor.ftensor4()
    h = tensor.ftensor3()
    iIdx = tensor.lmatrix()
    oIdx = tensor.lmatrix()

    o = sparse_block_dot(W, h, iIdx, b, oIdx)

    f = theano.function([W, h, iIdx, b, oIdx], o)
    assert hasattr(f.maker.fgraph.outputs[0].tag, 'trace')

    if theano.config.mode == "FAST_COMPILE":
        assert not f.maker.fgraph.toposort()[-1].op.inplace
    else:
        assert f.maker.fgraph.toposort()[-1].op.inplace 

def test_blocksparse_inplace_outer_opt():
    b = tensor.fmatrix()
    W = tensor.ftensor4()
    h = tensor.ftensor3()
    iIdx = tensor.lmatrix()
    oIdx = tensor.lmatrix()

    o = sparse_block_dot(W, h, iIdx, b, oIdx)

    theano.printing.debugprint(tensor.grad(o.sum(), wrt=W))

    f = theano.function([W, h, iIdx, b, oIdx],
                        [o, tensor.grad(o.sum(), wrt=W)])

    if theano.config.mode == "FAST_COMPILE":
        assert not f.maker.fgraph.toposort()[-1].op.inplace
    else:
        assert f.maker.fgraph.toposort()[-1].op.inplace 

def test_blocksparse_gpu_outer_opt():
    b = tensor.fmatrix()
    W = tensor.ftensor4()
    h = tensor.ftensor3()
    iIdx = tensor.lmatrix()
    oIdx = tensor.lmatrix()

    o = sparse_block_dot(W, h, iIdx, b, oIdx)

    f = theano.function([W, h, iIdx, b, oIdx], [o, tensor.grad(o.sum(),
                                                               wrt=W)],
                        mode=mode_with_gpu)

    assert isinstance(f.maker.fgraph.toposort()[-2].op, GpuSparseBlockOuter) 

def make_theano_batch(self, name=None, dtype=None, batch_size=None):
        if batch_size == 1:
            rval = tensor.lrow(name=name)
        else:
            rval = tensor.lmatrix(name=name)

        if theano.config.compute_test_value != 'off':
            if batch_size == 1:
                n = 1
            else:
                # TODO: try to extract constant scalar value from batch_size
                n = 4
            rval.tag.test_value = self.get_origin_batch(batch_size=n,
                                                        dtype=dtype)
        return rval 

def test_with_extra_dims_cross_entropy_3d():
    x = tensor.tensor3('x')
    y = tensor.lmatrix('y')
    brick = SoftmaxWithExtraDims()
    f = theano.function(
        [y, x], [brick.categorical_cross_entropy(y, x, extra_ndim=1)])
    assert_allclose(
        f([[0, 1], [2, 3]],
          [[[1, 2, 1, 2], [1, 2, 3, 4]],
           [[4, 3, 2, 1], [2, 2, 2, 2]]])[0],
        numpy.array([[2.0064, 2.44019],
                     [2.44019, 1.3863]]),
        rtol=1e-5) 

def test_lookup_table():
    lt = LookupTable(5, 3)
    lt.allocate()

    lt.W.set_value(numpy.arange(15).reshape(5, 3).astype(theano.config.floatX))

    x = tensor.lmatrix("x")
    y = lt.apply(x)
    f = theano.function([x], [y])

    x_val = [[1, 2], [0, 3]]
    desired = numpy.array([[[3, 4, 5], [6, 7, 8]], [[0, 1, 2], [9, 10, 11]]],
                          dtype=theano.config.floatX)
    assert_equal(f(x_val)[0], desired)

    # Test get_dim
    assert_equal(lt.get_dim(lt.apply.inputs[0]), 0)
    assert_equal(lt.get_dim(lt.apply.outputs[0]), lt.dim)
    assert_raises(ValueError, lt.get_dim, 'random_name')

    # Test feedforward interface
    assert lt.input_dim == 0
    assert lt.output_dim == 3
    lt.output_dim = 4
    assert lt.output_dim == 4

    def assign_input_dim():
        lt.input_dim = 11
    assert_raises(ValueError, assign_input_dim)
    lt.input_dim = 0 

def test_multMatVect():
    A1 = tensor.lmatrix('A1')
    s1 = tensor.ivector('s1')
    m1 = tensor.iscalar('m1')
    A2 = tensor.lmatrix('A2')
    s2 = tensor.ivector('s2')
    m2 = tensor.iscalar('m2')

    g0 = rng_mrg.DotModulo()(A1, s1, m1, A2, s2, m2)
    f0 = theano.function([A1, s1, m1, A2, s2, m2], g0)

    i32max = numpy.iinfo(numpy.int32).max

    A1 = numpy.random.randint(0, i32max, (3, 3)).astype('int64')
    s1 = numpy.random.randint(0, i32max, 3).astype('int32')
    m1 = numpy.asarray(numpy.random.randint(i32max), dtype="int32")
    A2 = numpy.random.randint(0, i32max, (3, 3)).astype('int64')
    s2 = numpy.random.randint(0, i32max, 3).astype('int32')
    m2 = numpy.asarray(numpy.random.randint(i32max), dtype="int32")

    f0.input_storage[0].storage[0] = A1
    f0.input_storage[1].storage[0] = s1
    f0.input_storage[2].storage[0] = m1
    f0.input_storage[3].storage[0] = A2
    f0.input_storage[4].storage[0] = s2
    f0.input_storage[5].storage[0] = m2

    r_a1 = rng_mrg.matVecModM(A1, s1, m1)
    r_a2 = rng_mrg.matVecModM(A2, s2, m2)
    f0.fn()
    r_b = f0.output_storage[0].value

    assert numpy.allclose(r_a1, r_b[:3])
    assert numpy.allclose(r_a2, r_b[3:]) 

def multMatVect(v, A, m1, B, m2):
    """
    Multiply the first half of v by A with a modulo of m1 and the second half
    by B with a modulo of m2.

    Notes
    -----
    The parameters of dot_modulo are passed implicitly because passing them
    explicitly takes more time than running the function's C-code.

    """
    if multMatVect.dot_modulo is None:
        A_sym = tensor.lmatrix('A')
        s_sym = tensor.ivector('s')
        m_sym = tensor.iscalar('m')
        A2_sym = tensor.lmatrix('A2')
        s2_sym = tensor.ivector('s2')
        m2_sym = tensor.iscalar('m2')
        o = DotModulo()(A_sym, s_sym, m_sym, A2_sym, s2_sym, m2_sym)
        multMatVect.dot_modulo = function(
            [A_sym, s_sym, m_sym, A2_sym, s2_sym, m2_sym], o, profile=False)

    # This way of calling the Theano fct is done to bypass Theano overhead.
    f = multMatVect.dot_modulo
    f.input_storage[0].storage[0] = A
    f.input_storage[1].storage[0] = v[:3]
    f.input_storage[2].storage[0] = m1
    f.input_storage[3].storage[0] = B
    f.input_storage[4].storage[0] = v[3:]
    f.input_storage[5].storage[0] = m2
    f.fn()
    r = f.output_storage[0].storage[0]

    return r 

def test_correct_solution(self):
        x = tensor.lmatrix()
        y = tensor.lmatrix()
        z = tensor.lscalar()
        b = theano.tensor.nlinalg.lstsq()(x, y, z)
        f = function([x, y, z], b)
        TestMatrix1 = numpy.asarray([[2, 1], [3, 4]])
        TestMatrix2 = numpy.asarray([[17, 20], [43, 50]])
        TestScalar = numpy.asarray(1)
        f = function([x, y, z], b)
        m = f(TestMatrix1, TestMatrix2, TestScalar)
        self.assertTrue(numpy.allclose(TestMatrix2, numpy.dot(TestMatrix1, m[0]))) 

def test_blocksparse_inplace_gemv_opt():
    b = tensor.fmatrix()
    W = tensor.ftensor4()
    h = tensor.ftensor3()
    iIdx = tensor.lmatrix()
    oIdx = tensor.lmatrix()

    o = sparse_block_dot(W, h, iIdx, b, oIdx)

    f = theano.function([W, h, iIdx, b, oIdx], o)

    if theano.config.mode == "FAST_COMPILE":
        assert not f.maker.fgraph.toposort()[-1].op.inplace
    else:
        assert f.maker.fgraph.toposort()[-1].op.inplace 

def __init__(self, dim, **kwargs):
        super(LookupBottom, self).__init__(**kwargs)
        self.dim = dim

        self.mask = tensor.matrix('inputs_mask')
        self.batch_inputs = {
            'inputs': tensor.lmatrix('inputs')}
        self.single_inputs = {
            'inputs': tensor.lvector('inputs')}

        self.children = [LookupTable(self.input_num_chars['inputs'], self.dim)] 

def test_multMatVect():
    A1 = tensor.lmatrix('A1')
    s1 = tensor.ivector('s1')
    m1 = tensor.iscalar('m1')
    A2 = tensor.lmatrix('A2')
    s2 = tensor.ivector('s2')
    m2 = tensor.iscalar('m2')

    g0 = rng_mrg.DotModulo()(A1, s1, m1, A2, s2, m2)
    f0 = theano.function([A1, s1, m1, A2, s2, m2], g0)

    i32max = numpy.iinfo(numpy.int32).max

    A1 = numpy.random.randint(0, i32max, (3, 3)).astype('int64')
    s1 = numpy.random.randint(0, i32max, 3).astype('int32')
    m1 = numpy.asarray(numpy.random.randint(i32max), dtype="int32")
    A2 = numpy.random.randint(0, i32max, (3, 3)).astype('int64')
    s2 = numpy.random.randint(0, i32max, 3).astype('int32')
    m2 = numpy.asarray(numpy.random.randint(i32max), dtype="int32")

    f0.input_storage[0].storage[0] = A1
    f0.input_storage[1].storage[0] = s1
    f0.input_storage[2].storage[0] = m1
    f0.input_storage[3].storage[0] = A2
    f0.input_storage[4].storage[0] = s2
    f0.input_storage[5].storage[0] = m2

    r_a1 = rng_mrg.matVecModM(A1, s1, m1)
    r_a2 = rng_mrg.matVecModM(A2, s2, m2)
    f0.fn()
    r_b = f0.output_storage[0].value

    assert numpy.allclose(r_a1, r_b[:3])
    assert numpy.allclose(r_a2, r_b[3:]) 

def test_blocksparse_gpu_gemv_opt():
    b = tensor.fmatrix()
    W = tensor.ftensor4()
    h = tensor.ftensor3()
    iIdx = tensor.lmatrix()
    oIdx = tensor.lmatrix()

    o = sparse_block_dot(W, h, iIdx, b, oIdx)

    f = theano.function([W, h, iIdx, b, oIdx], o, mode=mode_with_gpu)

    assert sum(1 for n in f.maker.fgraph.apply_nodes
               if isinstance(n.op, GpuSparseBlockGemv)) == 1 

def multMatVect(v, A, m1, B, m2):
    # TODO : need description for parameter and return
    """
    Multiply the first half of v by A with a modulo of m1 and the second half
    by B with a modulo of m2.

    Notes
    -----
    The parameters of dot_modulo are passed implicitly because passing them
    explicitly takes more time than running the function's C-code.

    """
    if multMatVect.dot_modulo is None:
        A_sym = tensor.lmatrix('A')
        s_sym = tensor.ivector('s')
        m_sym = tensor.iscalar('m')
        A2_sym = tensor.lmatrix('A2')
        s2_sym = tensor.ivector('s2')
        m2_sym = tensor.iscalar('m2')
        o = DotModulo()(A_sym, s_sym, m_sym, A2_sym, s2_sym, m2_sym)
        multMatVect.dot_modulo = function(
            [A_sym, s_sym, m_sym, A2_sym, s2_sym, m2_sym], o, profile=False)

    # This way of calling the Theano fct is done to bypass Theano overhead.
    f = multMatVect.dot_modulo
    f.input_storage[0].storage[0] = A
    f.input_storage[1].storage[0] = v[:3]
    f.input_storage[2].storage[0] = m1
    f.input_storage[3].storage[0] = B
    f.input_storage[4].storage[0] = v[3:]
    f.input_storage[5].storage[0] = m2
    f.fn()
    r = f.output_storage[0].storage[0]

    return r 

def test_correct_solution(self):
        x = tensor.lmatrix()
        y = tensor.lmatrix()
        z = tensor.lscalar()
        b = theano.tensor.nlinalg.lstsq()(x, y, z)
        f = function([x, y, z], b)
        TestMatrix1 = numpy.asarray([[2, 1], [3, 4]])
        TestMatrix2 = numpy.asarray([[17, 20], [43, 50]])
        TestScalar = numpy.asarray(1)
        f = function([x, y, z], b)
        m = f(TestMatrix1, TestMatrix2, TestScalar)
        self.assertTrue(numpy.allclose(TestMatrix2, numpy.dot(TestMatrix1, m[0]))) 

def Xtest_blocksparse_grad_merge(self):
        b = tensor.fmatrix()
        h = tensor.ftensor3()
        iIdx = tensor.lmatrix()
        oIdx = tensor.lmatrix()

        W_val, h_val, iIdx_val, b_val, oIdx_val = self.gemv_data()
        W = float32_shared_constructor(W_val)

        o = gpu_sparse_block_gemv(b.take(oIdx, axis=0), W, h, iIdx, oIdx)
        gW = theano.grad(o.sum(), W)

        lr = numpy.asarray(0.05, dtype='float32')

        upd = W - lr * gW

        f1 = theano.function([h, iIdx, b, oIdx], updates=[(W, upd)],
                             mode=mode_with_gpu)

        # Make sure the lr update was merged.
        assert isinstance(f1.maker.fgraph.outputs[0].owner.op,
                          GpuSparseBlockOuter)

        # Exclude the merge optimizations.
        mode = mode_with_gpu.excluding('local_merge_blocksparse_alpha')
        mode = mode.excluding('local_merge_blocksparse_output')

        f2 = theano.function([h, iIdx, b, oIdx], updates=[(W, upd)], mode=mode)

        # Make sure the lr update is not merged.
        assert not isinstance(f2.maker.fgraph.outputs[0].owner.op,
                              GpuSparseBlockOuter)

        f2(h_val, iIdx_val, b_val, oIdx_val)
        W_ref = W.get_value()

        # reset the var
        W.set_value(W_val)
        f1(h_val, iIdx_val, b_val, oIdx_val)
        W_opt = W.get_value()

        utt.assert_allclose(W_ref, W_opt) 

def Xtest_blocksparse_grad_merge(self):
        b = tensor.fmatrix()
        h = tensor.ftensor3()
        iIdx = tensor.lmatrix()
        oIdx = tensor.lmatrix()

        W_val, h_val, iIdx_val, b_val, oIdx_val = self.gemv_data()
        W = float32_shared_constructor(W_val)

        o = gpu_sparse_block_gemv(b.take(oIdx, axis=0), W, h, iIdx, oIdx)
        gW = theano.grad(o.sum(), W)

        lr = numpy.asarray(0.05, dtype='float32')

        upd = W - lr * gW

        f1 = theano.function([h, iIdx, b, oIdx], updates=[(W, upd)],
                             mode=mode_with_gpu)

        # Make sure the lr update was merged.
        assert isinstance(f1.maker.fgraph.outputs[0].owner.op,
                          GpuSparseBlockOuter)

        # Exclude the merge optimizations.
        mode = mode_with_gpu.excluding('local_merge_blocksparse_alpha')
        mode = mode.excluding('local_merge_blocksparse_output')

        f2 = theano.function([h, iIdx, b, oIdx], updates=[(W, upd)], mode=mode)

        # Make sure the lr update is not merged.
        assert not isinstance(f2.maker.fgraph.outputs[0].owner.op,
                              GpuSparseBlockOuter)

        f2(h_val, iIdx_val, b_val, oIdx_val)
        W_ref = W.get_value()

        # reset the var
        W.set_value(W_val)
        f1(h_val, iIdx_val, b_val, oIdx_val)
        W_opt = W.get_value()

        utt.assert_allclose(W_ref, W_opt) 

def __init__(self, data, config, fast_predict=False):
    self.embedding_shapes = data.embedding_shapes;
    self.lstm_type = config.lstm_cell  
    self.lstm_hidden_size = int(config.lstm_hidden_size)
    self.num_lstm_layers = int(config.num_lstm_layers)
    self.max_grad_norm = float(config.max_grad_norm)

    self.vocab_size = data.word_dict.size()
    self.label_space_size = data.label_dict.size()
    self.unk_id = data.unk_id
    
    # Initialize layers and parameters
    self.embedding_layer = EmbeddingLayer(data.embedding_shapes, data.embeddings)    
    self.params = [p for p in self.embedding_layer.params]
    
    self.rnn_layers = [None] * self.num_lstm_layers
    for l in range(self.num_lstm_layers):
      input_dim = self.embedding_layer.output_size if l == 0 else self.lstm_hidden_size
      input_dropout = config.input_dropout_prob if (config.per_layer_dropout or l == 0) else 0.0
      recurrent_dropout = config.recurrent_dropout_prob
      
      self.rnn_layers[l] = get_rnn_layer(self.lstm_type)(input_dim,
                                 self.lstm_hidden_size,
                                 input_dropout_prob=input_dropout,
                                 recurrent_dropout_prob=recurrent_dropout,
                                 fast_predict=fast_predict,
                                 prefix='lstm_{}'.format(l))
      print (self.rnn_layers[l])
      self.params.extend(self.rnn_layers[l].params)
    
    self.softmax_layer = SoftmaxLayer(self.lstm_hidden_size, self.label_space_size)
    self.params.extend(self.softmax_layer.params)
    
    # Build model
    # Shape of x: [seq_len, batch_size, num_features]
    self.x0 = tensor.ltensor3('x')
    self.y0 = tensor.lmatrix('y')
    self.mask0 = tensor.matrix('mask', dtype=floatX)
    self.is_train = tensor.bscalar('is_train')
    
    self.x = self.x0.dimshuffle(1, 0, 2)
    self.y = self.y0.dimshuffle(1, 0)
    self.mask = self.mask0.dimshuffle(1, 0) 
    
    self.inputs = [None] * (self.num_lstm_layers + 1)
    self.inputs[0] = self.embedding_layer.connect(self.x)
    self.rev_mask = self.mask[::-1]
    
    for l, rnn in enumerate(self.rnn_layers):
      outputs = rnn.connect(self.inputs[l],
                  self.mask if l % 2 == 0 else self.rev_mask,
                  self.is_train)
      self.inputs[l+1] = outputs[::-1]
     
    self.scores, self.pred = self.softmax_layer.connect(self.inputs[-1])
    self.pred0 = self.pred.reshape([self.x.shape[0], self.x.shape[1]]).dimshuffle(1, 0) 

def _RiskBackpropagate(Model, Features):
	"""
	Generates partial derivatives of input features in a neural network model.
	These represent the rate of change of risk with respect to each input
	feature.

	Parameters:
	----------
	Model : class
	A fine tuned model generated by training.

	Features : array_like
	A 1 x P numpy array of features corresponding to one sample.

	Output:
	----------
	Gradient : array_like
	A 1
	an array of the feature weights.
	"""

	# define partial derivative
	X = T.matrix('X')
	AtRisk = T.ivector('AtRisk')
	Observed = T.ivector('Observed')
	Is_train = T.scalar('Is_train', dtype='int32')
	masks = T.lmatrix('mask_' + str(0))
	partial_derivative = th.function(on_unused_input='ignore',
			inputs=[X, AtRisk, Observed, Is_train, masks],
			outputs=T.grad(Model.risk_layer.output[0],
				Model.x),
			givens={Model.x: X, Model.o: AtRisk,
				Model.at_risk: Observed,
				Model.is_train: Is_train,
				Model.masks[0]: masks},
			name='partial_derivative')

	# define parameters for risk and calculate partial
	sample_O = np.array([0]).astype(np.int32)
	sample_T = np.array([0]).astype(np.int32)
	# Create dummy masks for graph
	dummy_masks = np.ones((1, Model.n_hidden), dtype='int64')
	Gradient = partial_derivative(Features, sample_O, sample_T, 0, dummy_masks)

	return Gradient 

def _RiskBackpropagate(Model, Features):
	"""
	Generates partial derivatives of input features in a neural network model.
	These represent the rate of change of risk with respect to each input
	feature.

	Parameters:
	----------
	Model : class
	A fine tuned model generated by training.

	Features : array_like
	A 1 x P numpy array of features corresponding to one sample.

	Output:
	----------
	Gradient : array_like
	A 1
	an array of the feature weights.
	"""

	# define partial derivative
	X = T.matrix('X')
	AtRisk = T.ivector('AtRisk')
	Observed = T.ivector('Observed')
	Is_train = T.scalar('Is_train', dtype='int32')
	masks = T.lmatrix('mask_' + str(0))
	partial_derivative = th.function(on_unused_input='ignore',
			inputs=[X, AtRisk, Observed, Is_train, masks],
			outputs=T.grad(Model.risk_layer.output[0],
				Model.x),
			givens={Model.x: X, Model.o: AtRisk,
				Model.at_risk: Observed,
				Model.is_train: Is_train,
				Model.masks[0]: masks},
			name='partial_derivative')

	# define parameters for risk and calculate partial
	sample_O = np.array([0]).astype(np.int32)
	sample_T = np.array([0]).astype(np.int32)
	# Create dummy masks for graph
	dummy_masks = np.ones((1, Model.n_hidden), dtype='int64')
	Gradient = partial_derivative(Features, sample_O, sample_T, 0, dummy_masks)

	return Gradient 

def test_np_format_as_index2vector():
    # Test 5 random batches for shape, number of non-zeros
    for _ in xrange(5):
        max_labels = np.random.randint(2, 10)
        batch_size = np.random.randint(1, 10)
        labels = np.random.randint(1, 10)
        batch = np.random.random_integers(max_labels - 1,
                                          size=(batch_size, labels))
        index_space = IndexSpace(dim=labels, max_labels=max_labels)
        vector_space_merge = VectorSpace(dim=max_labels)
        vector_space_concatenate = VectorSpace(dim=max_labels * labels)
        merged = index_space.np_format_as(batch, vector_space_merge)
        concatenated = index_space.np_format_as(batch,
                                                vector_space_concatenate)
        assert merged.shape == (batch_size, max_labels)
        assert concatenated.shape == (batch_size, max_labels * labels)
        assert np.count_nonzero(merged) <= batch.size
        assert np.count_nonzero(concatenated) == batch.size
        assert np.all(np.unique(concatenated) == np.array([0, 1]))
    # Make sure Theano variables give the same result
    batch = tensor.lmatrix('batch')
    single = tensor.lvector('single')
    batch_size = np.random.randint(1, 10)
    np_batch = np.random.random_integers(max_labels - 1,
                                         size=(batch_size, labels))
    np_single = np.random.random_integers(max_labels - 1,
                                          size=(labels))
    f_batch_merge = theano.function(
        [batch], index_space._format_as_impl(False, batch, vector_space_merge)
    )
    f_batch_concatenate = theano.function(
        [batch], index_space._format_as_impl(False, batch,
                                             vector_space_concatenate)
    )
    f_single_merge = theano.function(
        [single], index_space._format_as_impl(False, single,
                                              vector_space_merge)
    )
    f_single_concatenate = theano.function(
        [single], index_space._format_as_impl(False, single,
                                              vector_space_concatenate)
    )
    np.testing.assert_allclose(
        f_batch_merge(np_batch),
        index_space._format_as_impl(True, np_batch, vector_space_merge)
    )
    np.testing.assert_allclose(
        f_batch_concatenate(np_batch),
        index_space._format_as_impl(True, np_batch, vector_space_concatenate)
    )
    np.testing.assert_allclose(
        f_single_merge(np_single),
        index_space._format_as_impl(True, np_single, vector_space_merge)
    )
    np.testing.assert_allclose(
        f_single_concatenate(np_single),
        index_space._format_as_impl(True, np_single, vector_space_concatenate)
    ) 

def test_matrixmul():
    """
    Tests for projection
    """
    rng = np.random.RandomState(222)
    dtypes = [
        'int16', 'int32', 'int64'
    ]
    tensor_x = [
        tensor.wmatrix(),
        tensor.imatrix(),
        tensor.lmatrix(),
        tensor.wvector(),
        tensor.ivector(),
        tensor.lvector()
    ]
    np_W, np_x = [], []
    for dtype in dtypes:
        np_W.append(rng.rand(10, np.random.randint(1, 10)))
        np_x.append(rng.randint(
            0, 10, (rng.random_integers(5),
                    rng.random_integers(5))
        ).astype(dtype))
    for dtype in dtypes:
        np_W.append(rng.rand(10, np.random.randint(1, 10)))
        np_x.append(
            rng.randint(0, 10, (rng.random_integers(5),)).astype(dtype)
        )

    tensor_W = [sharedX(W) for W in np_W]
    matrixmul = [MatrixMul(W) for W in tensor_W]
    assert all(mm.get_params()[0] == W for mm, W in zip(matrixmul, tensor_W))

    fn = [theano.function([x], mm.project(x))
          for x, mm in zip(tensor_x, matrixmul)]
    for W, x, f in zip(np_W, np_x, fn):
        W_x = W[x]
        if x.ndim == 2:
            W_x = W_x.reshape((W_x.shape[0], np.prod(W_x.shape[1:])))
        else:
            W_x = W_x.flatten()
        np.testing.assert_allclose(f(x), W_x) 

def test_matrixmul():
    """
    Tests matrix multiplication for a range of different
    dtypes. Checks both normal and transpose multiplication
    using randomly generated matrices.
    """
    rng = np.random.RandomState(222)
    dtypes = [
        'int16', 'int32', 'int64', 'float64', 'float32'
    ]
    tensor_x = [
        tensor.wmatrix(),
        tensor.imatrix(),
        tensor.lmatrix(),
        tensor.dmatrix(),
        tensor.fmatrix()
    ]
    np_W, np_x, np_x_T = [], [], []
    for dtype in dtypes:
        if 'int' in dtype:
            np_W.append(rng.randint(
                -10, 10, rng.random_integers(5, size=2)
            ).astype(dtype))
            np_x.append(rng.randint(
                -10, 10, (rng.random_integers(5),
                          np_W[-1].shape[0])
            ).astype(dtype))
            np_x_T.append(rng.randint(
                -10, 10, (rng.random_integers(5),
                          np_W[-1].shape[1])
            ).astype(dtype))
        elif 'float' in dtype:
            np_W.append(rng.uniform(
                -1, 1, rng.random_integers(5, size=2)
            ).astype(dtype))
            np_x.append(rng.uniform(
                -10, 10, (rng.random_integers(5),
                          np_W[-1].shape[0])
            ).astype(dtype))
            np_x.append(rng.uniform(
                -10, 10, (rng.random_integers(5),
                          np_W[-1].shape[1])
            ).astype(dtype))
        else:
            assert False

    def sharedW(value, dtype):
        return theano.shared(theano._asarray(value, dtype=dtype))
    tensor_W = [sharedW(W, dtype) for W in np_W]
    matrixmul = [MatrixMul(W) for W in tensor_W]
    assert all(mm.get_params()[0] == W for mm, W in zip(matrixmul, tensor_W))

    fn = [theano.function([x], mm.lmul(x))
          for x, mm in zip(tensor_x, matrixmul)]
    fn_T = [theano.function([x], mm.lmul_T(x))
            for x, mm in zip(tensor_x, matrixmul)]
    for W, x, x_T, f, f_T in zip(np_W, np_x, np_x_T, fn, fn_T):
        np.testing.assert_allclose(f(x), np.dot(x, W))
        np.testing.assert_allclose(f_T(x_T), np.dot(x_T, W.T)) 

def main(save_to, num_epochs):
    mlp = MLP([Tanh(), Softmax()], [784, 100, 10],
              weights_init=IsotropicGaussian(0.01),
              biases_init=Constant(0))
    mlp.initialize()
    x = tensor.matrix('features')
    y = tensor.lmatrix('targets')
    probs = mlp.apply(x)
    cost = CategoricalCrossEntropy().apply(y.flatten(), probs)
    error_rate = MisclassificationRate().apply(y.flatten(), probs)

    cg = ComputationGraph([cost])
    W1, W2 = VariableFilter(roles=[WEIGHT])(cg.variables)
    cost = cost + .00005 * (W1 ** 2).sum() + .00005 * (W2 ** 2).sum()
    cost.name = 'final_cost'

    mnist_train = MNIST(("train",))
    mnist_test = MNIST(("test",))

    algorithm = GradientDescent(
        cost=cost, parameters=cg.parameters,
        step_rule=Scale(learning_rate=0.1))
    extensions = [Timing(),
                  FinishAfter(after_n_epochs=num_epochs),
                  DataStreamMonitoring(
                      [cost, error_rate],
                      Flatten(
                          DataStream.default_stream(
                              mnist_test,
                              iteration_scheme=SequentialScheme(
                                  mnist_test.num_examples, 500)),
                          which_sources=('features',)),
                      prefix="test"),
                  TrainingDataMonitoring(
                      [cost, error_rate,
                       aggregation.mean(algorithm.total_gradient_norm)],
                      prefix="train",
                      after_epoch=True),
                  Checkpoint(save_to),
                  Printing()]

    if BLOCKS_EXTRAS_AVAILABLE:
        extensions.append(Plot(
            'MNIST example',
            channels=[
                ['test_final_cost',
                 'test_misclassificationrate_apply_error_rate'],
                ['train_total_gradient_norm']]))

    main_loop = MainLoop(
        algorithm,
        Flatten(
            DataStream.default_stream(
                mnist_train,
                iteration_scheme=SequentialScheme(
                    mnist_train.num_examples, 50)),
            which_sources=('features',)),
        model=Model(cost),
        extensions=extensions)

    main_loop.run()