Python scipy.sparse.csc_matrix() Examples

def test_equality_case(self):
        """
        Test assuring normal behaviour when values
        in the matrices are equal
        """

        scipy_ver = [int(n) for n in scipy.__version__.split('.')[:2]]

        if (bool(scipy_ver < [0, 13])):
            raise SkipTest("comparison operators need newer release of scipy")

        x = sparse.csc_matrix()
        y = theano.tensor.matrix()

        m1 = sp.csc_matrix((2, 2), dtype=theano.config.floatX)
        m2 = numpy.asarray([[0, 0], [0, 0]], dtype=theano.config.floatX)

        for func in self.testsDic:

            op = func(y, x)
            f = theano.function([y, x], op)

            self.assertTrue(numpy.array_equal(f(m2, m1),
                                              self.testsDic[func](m2, m1))) 

def perform(self, node, inp, outputs):
        (out,) = outputs
        x = inp[0]
        ind1 = inp[1]
        ind2 = inp[2]
        gz = inp[3]

        if x.format in ["csr"]:
            y = scipy.sparse.csr_matrix((x.shape[0], x.shape[1]))
        else:
            y = scipy.sparse.csc_matrix((x.shape[0], x.shape[1]))
        z = 0
        for z in range(0, len(ind1)):
            y[(ind1[z], ind2[z])] = gz[z]

        out[0] = y 

def perform(self, node, inputs, outputs):
        # for efficiency, if remap does nothing, then do not apply it
        (data, indices, indptr, shape) = inputs
        (out,) = outputs

        if len(shape) != 2:
            raise ValueError('Shape should be an array of length 2')
        if data.shape != indices.shape:
            errmsg = ('Data (shape ' + repr(data.shape) +
                      ' must have the same number of elements ' +
                      'as indices (shape' + repr(indices.shape) +
                      ')')
            raise ValueError(errmsg)
        if self.format == 'csc':
            out[0] = scipy.sparse.csc_matrix((data, indices.copy(),
                                              indptr.copy()),
                                             numpy.asarray(shape), copy=False)
        else:
            assert self.format == 'csr'
            out[0] = scipy.sparse.csr_matrix((data, indices.copy(),
                                              indptr.copy()), shape.copy(),
                                             copy=False) 

def perform(self, node, inputs, outputs):
        (x, s) = inputs
        (z,) = outputs
        M, N = x.shape
        assert x.format == 'csc'
        assert s.shape == (M,)

        indices = x.indices
        indptr = x.indptr

        y_data = x.data.copy()

        for j in xrange(0, N):
            for i_idx in xrange(indptr[j], indptr[j + 1]):
                y_data[i_idx] *= s[indices[i_idx]]

        z[0] = scipy.sparse.csc_matrix((y_data, indices, indptr), (M, N)) 

def create_matrix(dim, alpha=0.95, smallest_coef=0.1, largest_coef=.9):
    ''' Based o scikit-learn make_sparse_spd_matrix'''
    chol = -np.eye(dim)
    aux = np.random.rand(dim, dim)
    aux[aux < alpha] = 0
    aux[aux > alpha] = (smallest_coef
                        + (largest_coef - smallest_coef)
                        * np.random.rand(np.sum(aux > alpha)))
    aux = np.tril(aux, k=-1)

    # Permute the lines: we don't want to have asymmetries in the final
    # SPD matrix
    permutation = np.random.permutation(dim)
    aux = aux[permutation].T[permutation]
    chol += aux
    A = sp.csc_matrix(np.dot(chol.T, chol))

    x = np.random.rand(dim)
    b = A.dot(x)

    return A,b,x 

def encode_sentence(self, sent, maxlen, startsig=False, endsig=False):
        """ Encode one sentence to a sparse matrix, with each row the expanded vector of each character.

        :param sent: sentence
        :param maxlen: maximum length of the sentence
        :param startsig: signal character at the beginning of the sentence (Default: False)
        :param endsig: signal character at the end of the sentence (Default: False)
        :return: matrix representing the sentence
        :type sent: str
        :type maxlen: int
        :type startsig: bool
        :type endsig: bool
        :rtype: scipy.sparse.csc_matrix
        """
        cor_sent = (self.signalchar if startsig else '') + sent[:min(maxlen, len(sent))] + (self.signalchar if endsig else '')
        sent_vec = self.calculate_prelim_vec(cor_sent).tocsc()
        if sent_vec.shape[0] == maxlen + startsig + endsig:
            return sent_vec
        else:
            return csc_matrix((sent_vec.data, sent_vec.indices, sent_vec.indptr),
                              shape=(maxlen + startsig + endsig, sent_vec.shape[1]),
                              dtype=np.float64) 

def encode_sentences(self, sentences, maxlen, sparse=True, startsig=False, endsig=False):
        """ Encode many sentences into a rank-3 tensor.

        :param sentences: sentences
        :param maxlen: maximum length of one sentence
        :param sparse: whether to return a sparse matrix (Default: True)
        :param startsig: signal character at the beginning of the sentence (Default: False)
        :param endsig: signal character at the end of the sentence (Default: False)
        :return: rank-3 tensor of the sentences
        :type sentences: list
        :type maxlen: int
        :type sparse: bool
        :type startsig: bool
        :type endsig: bool
        :rtype: scipy.sparse.csc_matrix or numpy.array
        """
        encode_sent_func = partial(self.encode_sentence, startsig=startsig, endsig=endsig, maxlen=maxlen)
        list_encoded_sentences_map = map(encode_sent_func, sentences)
        if sparse:
            return list(list_encoded_sentences_map)
        else:
            return np.array([sparsevec.toarray() for sparsevec in list_encoded_sentences_map]) 

def prepare_solver(self):
        '''Prepares the object to solve FEM systems
        Note
        -------
        After running this method, do NOT change any attributes of the class!
        '''
        logger.info(f'Using solver options: {self._solver_options}')
        A = sparse.csc_matrix(self.A, copy=True)
        A.sort_indices()
        dof_map = copy.deepcopy(self.dof_map)
        if self.dirichlet is not None:
            A, dof_map = self.dirichlet.apply_to_matrix(A, dof_map)

        if self._solver_options == 'pardiso':
            self._solver = pardiso.Solver(A)
        else:
            self._A_reduced = A  # We need to save this as PETSc does not copy the vectors
            self._solver = petsc_solver.Solver(self._solver_options, A) 

def load_matlab_file(path_file, name_field):
    """
    load '.mat' files
    inputs:
        path_file, string containing the file path
        name_field, string containig the field name (default='shape')
    warning:
        '.mat' files should be saved in the '-v7.3' format
    """
    db = h5py.File(path_file, 'r')
    ds = db[name_field]
    try:
        if 'ir' in ds.keys():
            data = np.asarray(ds['data'])
            ir = np.asarray(ds['ir'])
            jc = np.asarray(ds['jc'])
            out = sp.csc_matrix((data, ir, jc)).astype(np.float32)
    except AttributeError:
        # Transpose in case is a dense matrix because of the row- vs column- major ordering between python and matlab
        out = np.asarray(ds).astype(np.float32).T

    db.close()

    return out 

def fit(self, X, y=None):
        mean = y.mean()
        lower = mean - self.margin
        upper = mean + self.margin
        ys = sparse.csc_matrix(y[:, np.newaxis])
        if self.weighted:
            x = X.multiply(ys).sum(axis=0)
            x = x / X.sum(axis=0)
        else:
            x = (X > 0)
            s = x.sum(axis=0)
            x = x.multiply(ys).sum(axis=0) / s
        x = np.array(x).flatten().astype('f4')
        mask1 = (x < lower)
        mask2 = (x > upper)
        self.mask = (mask1 + mask2).astype(bool)
        return self 

def test_structured_add_s_v(self):
        sp_types = {'csc': sp.csc_matrix,
                    'csr': sp.csr_matrix}

        for format in ['csr', 'csc']:
            for dtype in ['float32', 'float64']:
                x = theano.sparse.SparseType(format, dtype=dtype)()
                y = tensor.vector(dtype=dtype)
                f = theano.function([x, y], structured_add_s_v(x, y))

                spmat = sp_types[format](random_lil((4, 3), dtype, 3))
                spones = spmat.copy()
                spones.data = numpy.ones_like(spones.data)
                mat = numpy.asarray(numpy.random.rand(3), dtype=dtype)

                out = f(spmat, mat)

                utt.assert_allclose(as_ndarray(spones.multiply(spmat + mat)),
                                    out.toarray()) 

def test_mul_s_v(self):
        sp_types = {'csc': sp.csc_matrix,
                    'csr': sp.csr_matrix}

        for format in ['csr', 'csc']:
            for dtype in ['float32', 'float64']:
                x = theano.sparse.SparseType(format, dtype=dtype)()
                y = tensor.vector(dtype=dtype)
                f = theano.function([x, y], mul_s_v(x, y))

                spmat = sp_types[format](random_lil((4, 3), dtype, 3))
                mat = numpy.asarray(numpy.random.rand(3), dtype=dtype)

                out = f(spmat, mat)

                utt.assert_allclose(spmat.toarray() * mat, out.toarray()) 

def test_infer_shape(self):
        mat = (numpy.arange(12) + 1).reshape((4, 3))
        mat[0, 1] = mat[1, 0] = mat[2, 2] = 0

        x_csc = theano.sparse.csc_matrix(dtype=theano.config.floatX)
        mat_csc = sp.csc_matrix(mat, dtype=theano.config.floatX)
        self._compile_and_check([x_csc],
                                [Remove0()(x_csc)],
                                [mat_csc],
                                self.op_class)

        x_csr = theano.sparse.csr_matrix(dtype=theano.config.floatX)
        mat_csr = sp.csr_matrix(mat, dtype=theano.config.floatX)
        self._compile_and_check([x_csr],
                                [Remove0()(x_csr)],
                                [mat_csr],
                                self.op_class) 

def _read_hb_data(content, header):
    # XXX: look at a way to reduce memory here (big string creation)
    ptr_string = "".join([content.read(header.pointer_nbytes_full),
                           content.readline()])
    ptr = np.fromstring(ptr_string,
            dtype=int, sep=' ')

    ind_string = "".join([content.read(header.indices_nbytes_full),
                       content.readline()])
    ind = np.fromstring(ind_string,
            dtype=int, sep=' ')

    val_string = "".join([content.read(header.values_nbytes_full),
                          content.readline()])
    val = np.fromstring(val_string,
            dtype=header.values_dtype, sep=' ')

    try:
        return csc_matrix((val, ind-1, ptr-1),
                          shape=(header.nrows, header.ncols))
    except ValueError as e:
        raise e 

def test_size():
    """
    Ensure the `size` attribute of sparse matrices behaves as in numpy.
    """
    for sparse_type in ('csc_matrix', 'csr_matrix'):
        x = getattr(theano.sparse, sparse_type)()
        y = getattr(scipy.sparse, sparse_type)((5, 7)).astype(config.floatX)
        get_size = theano.function([x], x.size)

        def check():
            assert y.size == get_size(y)
        # We verify that the size is correctly updated as we store more data
        # into the sparse matrix (including zeros).
        check()
        y[0, 0] = 1
        check()
        y[0, 1] = 0
        check() 

def adult_dataload():
    from scipy.sparse import csc_matrix
    import scipy.io as sio
    import numpy as np
    filepath='D:/PostDocWork/LSML/RandomFourierFeatures/Datasets/adult/adult/adult123.mat'
    adult=sio.loadmat(filepath)
    Dummy=adult['XTrain']
    TrainData=csc_matrix(Dummy,shape=Dummy.shape).toarray()
    TrainData=TrainData.T
    train_label=np.squeeze(adult['yTrain'])
    Dummy1=adult['XTest']
    TestData = csc_matrix(Dummy1,shape=Dummy1.shape).toarray()
    TestData=TestData.T
    test_label = np.squeeze(adult['yTest'])
    del Dummy, Dummy1
    return TrainData, train_label, TestData, test_label 

def test_csc_dense(self):
        x = theano.sparse.csc_matrix('x')
        y = theano.tensor.matrix('y')
        v = theano.tensor.vector('v')

        for (x, y, x_v, y_v) in [(x, y, self.x_csc, self.y),
                                 (x, v, self.x_csc, self.v_100),
                                 (v, x, self.v_10, self.x_csc)]:

            f_a = theano.function([x, y], theano.sparse.dot(x, y))
            f_b = lambda x, y: x * y

            utt.assert_allclose(f_a(x_v, y_v), f_b(x_v, y_v))

            # Test infer_shape
            self._compile_and_check([x, y], [theano.sparse.dot(x, y)],
                                    [x_v, y_v],
                                    (Dot, Usmm, UsmmCscDense)) 

def ijcnn1_dataload():
    from scipy.sparse import csc_matrix
    import scipy.io as sio
    import numpy as np
    filepath='D:\PostDocWork\LSML\RandomFourierFeatures\Datasets\ijcnn1\ijcnn1_combined.mat'
    adult=sio.loadmat(filepath)
    Dummy=adult['Xtrain']
    TrainData=csc_matrix(Dummy,shape=Dummy.shape).toarray()
    train_label=np.squeeze(adult['ytrain'])
    Dummy1=adult['Xtest']
    TestData = csc_matrix(Dummy1,shape=Dummy1.shape).toarray()
    test_label = np.squeeze(adult['ytest'])
    Dummy2=adult['Xval']
    ValData = csc_matrix(Dummy2,shape=Dummy2.shape).toarray()
    val_label = np.squeeze(adult['yval'])
    return TrainData, train_label, TestData, test_label, ValData, val_label 

def census_dataload():
    from scipy.sparse import csc_matrix
    import scipy.io as sio
    import numpy as np
    filepath='D:/PostDocWork/LSML/RandomFourierFeatures/Datasets/census/census/census.mat'
    adult=sio.loadmat(filepath)
    Dummy=adult['Xtrain']
    TrainData=csc_matrix(Dummy,shape=Dummy.shape).toarray()
    TrainData=TrainData.T
    train_label=np.squeeze(adult['ytrain'])
    Dummy1=adult['Xtest']
    TestData = csc_matrix(Dummy1,shape=Dummy1.shape).toarray()
    TestData=TestData.T
    test_label = np.squeeze(adult['ytest'])
    del Dummy, Dummy1
    return TrainData, train_label, TestData, test_label 

def cpu_dataload():
    from scipy.sparse import csc_matrix
    import scipy.io as sio
    import numpy as np
    filepath='D:/PostDocWork/LSML/RandomFourierFeatures/Datasets/cpu/cpu/cpu.mat'
    adult=sio.loadmat(filepath)
    Dummy=adult['Xtrain']
    TrainData=csc_matrix(Dummy,shape=Dummy.shape).toarray()
    TrainData=TrainData.T
    train_label=np.squeeze(adult['ytrain'])
    Dummy1=adult['Xtest']
    TestData = csc_matrix(Dummy1,shape=Dummy1.shape).toarray()
    TestData=TestData.T
    test_label = np.squeeze(adult['ytest'])
    del Dummy, Dummy1
    return TrainData, train_label, TestData, test_label 

def setUp(self):
        super(DotTests, self).setUp()
        x_size = (10, 100)
        y_size = (100, 1000)
        utt.seed_rng()

        self.x_csr = scipy.sparse.csr_matrix(
            numpy.random.binomial(1, 0.5, x_size), dtype=theano.config.floatX)
        self.x_csc = scipy.sparse.csc_matrix(
            numpy.random.binomial(1, 0.5, x_size), dtype=theano.config.floatX)
        self.y = numpy.asarray(numpy.random.uniform(-1, 1, y_size),
                               dtype=theano.config.floatX)
        self.y_csr = scipy.sparse.csr_matrix(
            numpy.random.binomial(1, 0.5, y_size), dtype=theano.config.floatX)
        self.y_csc = scipy.sparse.csc_matrix(
            numpy.random.binomial(1, 0.5, y_size), dtype=theano.config.floatX)
        self.v_10 = numpy.asarray(numpy.random.uniform(-1, 1, 10),
                                  dtype=theano.config.floatX)
        self.v_100 = numpy.asarray(numpy.random.uniform(-1, 1, 100),
                                   dtype=theano.config.floatX) 

def test_dot_sparse_sparse(self):
        # test dot for 2 input sparse matrix
        sparse_dtype = 'float64'
        sp_mat = {'csc': sp.csc_matrix,
                  'csr': sp.csr_matrix,
                  'bsr': sp.csr_matrix}

        for sparse_format_a in ['csc', 'csr', 'bsr']:
            for sparse_format_b in ['csc', 'csr', 'bsr']:
                a = SparseType(sparse_format_a, dtype=sparse_dtype)()
                b = SparseType(sparse_format_b, dtype=sparse_dtype)()
                d = theano.dot(a, b)
                f = theano.function([a, b], theano.Out(d, borrow=True))
                topo = f.maker.fgraph.toposort()
                for M, N, K, nnz in [(4, 3, 2, 3),
                                     (40, 30, 20, 3),
                                     (40, 30, 20, 30),
                                     (400, 3000, 200, 6000),
                                 ]:
                    a_val = sp_mat[sparse_format_a](
                        random_lil((M, N), sparse_dtype, nnz))
                    b_val = sp_mat[sparse_format_b](
                        random_lil((N, K), sparse_dtype, nnz))
                    f(a_val, b_val) 

def test_csm(self):
        sp_types = {'csc': sp.csc_matrix,
                    'csr': sp.csr_matrix}

        for format in ['csc', 'csr']:
            for dtype in ['float32', 'float64']:
                x = tensor.tensor(dtype=dtype, broadcastable=(False,))
                y = tensor.ivector()
                z = tensor.ivector()
                s = tensor.ivector()
                f = theano.function([x, y, z, s], CSM(format)(x, y, z, s))

                spmat = sp_types[format](random_lil((4, 3), dtype, 3))

                res = f(spmat.data, spmat.indices, spmat.indptr,
                        numpy.asarray(spmat.shape, 'int32'))

                assert numpy.all(res.data == spmat.data)
                assert numpy.all(res.indices == spmat.indices)
                assert numpy.all(res.indptr == spmat.indptr)
                assert numpy.all(res.shape == spmat.shape) 

def test_local_csm_properties_csm():
    data = tensor.vector()
    indices, indptr, shape = (tensor.ivector(), tensor.ivector(),
                              tensor.ivector())
    mode = theano.compile.mode.get_default_mode()
    mode = mode.including("specialize", "local_csm_properties_csm")
    for CS, cast in [(sparse.CSC, sp.csc_matrix),
                     (sparse.CSR, sp.csr_matrix)]:
        f = theano.function([data, indices, indptr, shape],
                            sparse.csm_properties(
                                CS(data, indices, indptr, shape)),
                            mode=mode)
        assert not any(
            isinstance(node.op, (sparse.CSM, sparse.CSMProperties))
            for node in f.maker.fgraph.toposort())
        v = cast(random_lil((10, 40),
                            config.floatX, 3))
        f(v.data, v.indices, v.indptr, v.shape) 

def test_local_csm_grad_c():
    raise SkipTest("Opt disabled as it don't support unsorted indices")
    if not theano.config.cxx:
        raise SkipTest("G++ not available, so we need to skip this test.")
    data = tensor.vector()
    indices, indptr, shape = (tensor.ivector(), tensor.ivector(),
                              tensor.ivector())
    mode = theano.compile.mode.get_default_mode()

    if theano.config.mode == 'FAST_COMPILE':
        mode = theano.compile.Mode(linker='c|py', optimizer='fast_compile')

    mode = mode.including("specialize", "local_csm_grad_c")
    for CS, cast in [(sparse.CSC, sp.csc_matrix), (sparse.CSR, sp.csr_matrix)]:
        cost = tensor.sum(sparse.DenseFromSparse()(CS(data, indices, indptr, shape)))
        f = theano.function(
            [data, indices, indptr, shape],
            tensor.grad(cost, data),
            mode=mode)
        assert not any(isinstance(node.op, sparse.CSMGrad) for node
                       in f.maker.fgraph.toposort())
        v = cast(random_lil((10, 40),
                            config.floatX, 3))
        f(v.data, v.indices, v.indptr, v.shape) 

def test_csm_unsorted(self):
        """
        Test support for gradients of unsorted inputs.
        """
        sp_types = {'csc': sp.csc_matrix,
                    'csr': sp.csr_matrix}

        for format in ['csr', 'csc', ]:
            for dtype in ['float32', 'float64']:
                x = tensor.tensor(dtype=dtype, broadcastable=(False,))
                y = tensor.ivector()
                z = tensor.ivector()
                s = tensor.ivector()
                # Sparse advanced indexing produces unsorted sparse matrices
                a = sparse_random_inputs(format, (4, 3), out_dtype=dtype,
                                         unsorted_indices=True)[1][0]
                # Make sure it's unsorted
                assert not a.has_sorted_indices
                def my_op(x):
                    y = tensor.constant(a.indices)
                    z = tensor.constant(a.indptr)
                    s = tensor.constant(a.shape)
                    return tensor.sum(
                        dense_from_sparse(CSM(format)(x, y, z, s) * a))
                verify_grad_sparse(my_op, [a.data]) 

def test_csm_properties_grad(self):
        sp_types = {'csc': sp.csc_matrix,
                    'csr': sp.csr_matrix}

        for format in ['csc', 'csr']:
            for dtype in ['float32', 'float64']:
                spmat = sp_types[format](random_lil((4, 3), dtype, 3))

                verify_grad_sparse(lambda *x: CSMProperties()(*x)[0], [spmat],
                                   structured=True)

                verify_grad_sparse(lambda *x: CSMProperties()(*x)[1], [spmat],
                                   structured=True)

                verify_grad_sparse(lambda *x: CSMProperties()(*x)[2], [spmat],
                                   structured=True)

                verify_grad_sparse(lambda *x: CSMProperties()(*x)[2], [spmat],
                                   structured=True) 

def test_GetItemList(self):

        a, A = sparse_random_inputs('csr', (4, 5))
        b, B = sparse_random_inputs('csc', (4, 5))
        y = a[0][[0, 1, 2, 3, 1]]
        z = b[0][[0, 1, 2, 3, 1]]

        fa = theano.function([a[0]], y)
        fb = theano.function([b[0]], z)

        t_geta = fa(A[0]).todense()
        t_getb = fb(B[0]).todense()

        s_geta = scipy.sparse.csr_matrix(A[0])[[0, 1, 2, 3, 1]].todense()
        s_getb = scipy.sparse.csc_matrix(B[0])[[0, 1, 2, 3, 1]].todense()

        utt.assert_allclose(t_geta, s_geta)
        utt.assert_allclose(t_getb, s_getb) 

def to_csc(self):
        """Convert Dataset to scipy's Compressed Sparse Column matrix."""
        self._X_train = csc_matrix(self._X_train)
        self._X_test = csc_matrix(self._X_test) 

def test_structured_add_s_v_grad(self):
        sp_types = {'csc': sp.csc_matrix,
                    'csr': sp.csr_matrix}

        for format in ['csr', 'csc']:
            for dtype in ['float32', 'float64']:
                spmat = sp_types[format](random_lil((4, 3), dtype, 3))
                mat = numpy.asarray(numpy.random.rand(3), dtype=dtype)

                theano.sparse.verify_grad_sparse(structured_add_s_v,
                                                 [spmat, mat],
                                                 structured=True)