cern.colt.matrix.impl.DenseDoubleMatrix2D Java Examples

@Override
public void aggregate(double[][][] values) {
	result = null;
	int offset = super.calculateOffset(values,featureNameIndecis);
	int[][] featureIndecis = super.collapseFeatures(values,featureNameIndecis);
	result[0] = 0.0;
	windowLength = featureNameIndecis.length-offset;
	featureLength = featureIndecis[0].length;
	for (int i=offset;i<values.length;++i){
		for(int j=0;j<featureIndecis.length;++j){
			result[0] += values[i][featureIndecis[j][0]][featureIndecis[j][1]];
		}
	}		
	terms = new DenseDoubleMatrix2D(xDim*yDim,windowLength*featureLength);
	z = new DenseDoubleMatrix2D(1,featureLength);
	calcTerms(terms);
	result = ((new Algebra()).solve(terms,z)).viewRow(0).toArray();
}
 

@Override
public <U, I> Factorization<U, I> load(InputStream in, FastUserIndex<U> uIndex, FastItemIndex<I> iIndex) throws IOException {
    int K;
    DenseDoubleMatrix2D userMatrix;
    DenseDoubleMatrix2D itemMatrix;
    try (ZipInputStream zip = new ZipInputStream(in)) {
        zip.getNextEntry();
        BufferedReader reader = new BufferedReader(new InputStreamReader(zip));
        int numUsers = parseInt(reader.readLine());
        int numItems = parseInt(reader.readLine());
        K = parseInt(reader.readLine());
        zip.closeEntry();

        zip.getNextEntry();
        userMatrix = loadDenseDoubleMatrix2D(zip, numUsers, K);
        zip.closeEntry();

        zip.getNextEntry();
        itemMatrix = loadDenseDoubleMatrix2D(zip, numItems, K);
        zip.closeEntry();
    }

    return new Factorization<>(uIndex, iIndex, userMatrix, itemMatrix, K);
}
 

public double[] transformMatrix(double[][] inputMatrix, int dim) {
    Algebra algebra = new Algebra();

    DoubleMatrix2D H = new DenseDoubleMatrix2D(inputMatrix);
    RobustEigenDecomposition decomposition = new RobustEigenDecomposition(H);
    DoubleMatrix1D eigenvalues = decomposition.getRealEigenvalues();

    normalizeEigenvalues(eigenvalues);

    DoubleMatrix2D V = decomposition.getV();

    transformEigenvalues(eigenvalues);

    double[][] negativeHessianInverse = algebra.mult(
            algebra.mult(V, DoubleFactory2D.dense.diagonal(eigenvalues)),
            algebra.inverse(V)
    ).toArray();

    double[] massArray = new double[dim * dim];
    for (int i = 0; i < dim; i++) {
        System.arraycopy(negativeHessianInverse[i], 0, massArray, i * dim, dim);
    }
    return massArray;
}
 

@Override
public FastRecommendation getRecommendation(int uidx, IntStream candidates) {
    DoubleMatrix1D pu;

    pu = factorization.getUserVector(uidx2user(uidx));
    if (pu == null) {
        return new FastRecommendation(uidx, new ArrayList<>());
    }
    
    DenseDoubleMatrix2D q = factorization.getItemMatrix();
    
    List<Tuple2id> items = candidates
            .mapToObj(iidx -> tuple(iidx, q.viewRow(iidx).zDotProduct(pu)))
            .sorted(comparingDouble(Tuple2id::v2).reversed())
            .collect(toList());
    
    return new FastRecommendation(uidx, items);
}
 

private static DoubleMatrix2D getGt(final DenseDoubleMatrix2D p, final DenseDoubleMatrix2D q, double lambda) {
    final int K = p.columns();

    DenseDoubleMatrix2D A1 = new DenseDoubleMatrix2D(K, K);
    q.zMult(q, A1, 1.0, 0.0, true, false);
    for (int k = 0; k < K; k++) {
        A1.setQuick(k, k, lambda + A1.getQuick(k, k));
    }

    EigenvalueDecomposition eig = new EigenvalueDecomposition(A1);
    DoubleMatrix1D d = eig.getRealEigenvalues();
    DoubleMatrix2D gt = eig.getV();
    for (int k = 0; k < K; k++) {
        double a = sqrt(d.get(k));
        gt.viewColumn(k).assign(x -> a * x);
    }

    return gt;
}
 

@Override
public double error(DenseDoubleMatrix2D p, DenseDoubleMatrix2D q, FastPreferenceData<U, I> data) {
    // TODO: add regularization, unify with HKVFactorizer's error
    
    return data.getUidxWithPreferences().parallel().mapToDouble(uidx -> {
        DoubleMatrix1D pu = p.viewRow(uidx);
        DoubleMatrix1D su = q.zMult(pu, null);
        
        double err1 = data.getUidxPreferences(uidx).mapToDouble(iv -> {
            double rui = iv.v2;
            double sui = su.getQuick(iv.v1);
            double cui = confidence.applyAsDouble(rui);
            return cui * (rui - sui) * (rui - sui) - confidence.applyAsDouble(0) * sui * sui;
        }).sum();
        
        double err2 = confidence.applyAsDouble(0) * su.assign(x -> x * x).zSum();
        
        return (err1 + err2) / data.numItems();
    }).sum() / data.numUsers();

}
 

/**
 * Constructor that sets description, dependencies, and offsets from
 * FeatureExtractor
 */
public AreaPolynomialApproximationLogConstantQ() {
	String name = "2D Polynomial Approximation of Log of ConstantQ";
	String description = "coeffecients of 2D polynomial best describing the input matrix.";
	String[] attributes = new String[] { "horizontal size (window length)",
										"vertical size (number of feature dimensions)",
										"number of x (horizontal) terms",
										"number of y (vertical) terms" };

	definition = new FeatureDefinition(name, description, true, 0,
			attributes);
			
	dependencies = new String[windowLength];
	for (int i = 0; i < dependencies.length; ++i) {
		dependencies[i] = "Log of ConstantQ";
	}
	
	offsets = new int[windowLength];
	for (int i = 0; i < offsets.length; ++i) {
		offsets[i] = 0 - i;
	}
	
	terms = new DenseDoubleMatrix2D(windowLength*featureLength,k*l);
	z = new DenseDoubleMatrix2D(featureLength*windowLength,1);
	calcTerms(terms);
}
 

/**
 * Function that must be overridden to allow this feature to be set globally
 * by GlobalChange frame.
 * 
 * @param n
 *            the number of windows of offset to be used in calculating this
 *            feature
 */
public void setWindow(int n) throws Exception {
	if (n < 1) {
		throw new Exception(
				"Area Polynomial Approximation window length must be positive");
	} else {
		windowLength = n;
		dependencies = new String[windowLength];
		offsets = new int[windowLength];
		for (int i = 0; i < windowLength; ++i) {
			dependencies[i] = "Magnitude Spectrum";
			offsets[i] = 0 - i;
		}
		terms = new DenseDoubleMatrix2D(windowLength*featureLength,k*l);
		z = new DenseDoubleMatrix2D(1,featureLength*windowLength);
		calcTerms(terms);
	}
}
 

@Override
public double error(DenseDoubleMatrix2D p, DenseDoubleMatrix2D q, FastPreferenceData<U, I> data) {
    // TODO: add regularization
    
    return data.getUidxWithPreferences().parallel().mapToDouble(uidx -> {
        DoubleMatrix1D pu = p.viewRow(uidx);
        DoubleMatrix1D su = q.zMult(pu, null);
        
        double err1 = data.getUidxPreferences(uidx).mapToDouble(iv -> {
            double rui = iv.v2;
            double sui = su.getQuick(iv.v1);
            double cui = confidence.applyAsDouble(rui);
            return cui * (rui - sui) * (rui - sui) - confidence.applyAsDouble(0) * sui * sui;
        }).sum();
        
        double err2 = confidence.applyAsDouble(0) * su.assign(x -> x * x).zSum();
        
        return (err1 + err2) / data.numItems();
    }).sum() / data.numUsers();

}
 

/**
 * Constructor that sets description, dependencies, and offsets from
 * FeatureExtractor
 */
public AreaPolynomialApproximationConstantQMFCC() {
	String name = "2D Polynomial Approximation ConstantQ MFCC";
	String description = "coeffecients of 2D polynomial best describing the input matrtix.";
	String[] attributes = new String[] { "horizontal size (window length)",
										"vertical size (number of feature dimensions)",
										"number of x (horizontal) terms",
										"number of y (vertical) terms" };

	definition = new FeatureDefinition(name, description, true, 0,
			attributes);
			
	dependencies = new String[windowLength];
	for (int i = 0; i < dependencies.length; ++i) {
		dependencies[i] = "ConstantQ derived MFCCs";
	}
	
	offsets = new int[windowLength];
	for (int i = 0; i < offsets.length; ++i) {
		offsets[i] = 0 - i;
	}
	
	terms = new DenseDoubleMatrix2D(k*l,windowLength*featureLength);
	z = new DenseDoubleMatrix2D(1,featureLength*windowLength);
	calcTerms(terms);
}
 

/**
 * Function that must be overridden to allow this feature to be set globally
 * by GlobalChange frame.
 * 
 * @param n
 *            the number of windows of offset to be used in calculating this
 *            feature
 */
public void setWindow(int n) throws Exception {
	if (n < 1) {
		throw new Exception(
				"Area Polynomial Approximation window length must be positive");
	} else {
		windowLength = n;
		dependencies = new String[windowLength];
		offsets = new int[windowLength];
		for (int i = 0; i < windowLength; ++i) {
			dependencies[i] = "Magnitude Spectrum";
			offsets[i] = 0 - i;
		}
		terms = new DenseDoubleMatrix2D(k*l,windowLength*featureLength);
		z = new DenseDoubleMatrix2D(1,featureLength*windowLength);
		calcTerms(terms);
	}
}
 

/**
 * Constructor that sets description, dependencies, and offsets from
 * FeatureExtractor
 */
public AreaPolynomialApproximation() {
	String name = "2D Polynomial Approximation";
	String description = "coeffecients of 2D polynomial best describing the input matrtix.";
	String[] attributes = new String[] { "horizontal size (window length)",
										"vertical size (number of feature dimensions)",
										"number of x (horizontal) terms",
										"number of y (vertical) terms" };

	definition = new FeatureDefinition(name, description, true, 0,
			attributes);
			
	dependencies = new String[windowLength];
	for (int i = 0; i < dependencies.length; ++i) {
		dependencies[i] = "Magnitude Spectrum";
	}
	
	offsets = new int[windowLength];
	for (int i = 0; i < offsets.length; ++i) {
		offsets[i] = 0 - i;
	}
	
	terms = new DenseDoubleMatrix2D(k*l,windowLength*featureLength);
	z = new DenseDoubleMatrix2D(1,featureLength*windowLength);
	calcTerms(terms);
}
 

/**
 * Function that must be overridden to allow this feature to be set globally
 * by GlobalChange frame.
 * 
 * @param n
 *            the number of windows of offset to be used in calculating this
 *            feature
 */
public void setWindow(int n) throws Exception {
	if (n < 1) {
		throw new Exception(
				"Area Polynomial Approximation window length must be positive");
	} else {
		windowLength = n;
		dependencies = new String[windowLength];
		offsets = new int[windowLength];
		for (int i = 0; i < windowLength; ++i) {
			dependencies[i] = "Magnitude Spectrum";
			offsets[i] = 0 - i;
		}
		terms = new DenseDoubleMatrix2D(k*l,windowLength*featureLength);
		z = new DenseDoubleMatrix2D(1,featureLength*windowLength);
		calcTerms(terms);
	}
}
 

/**
 * Constructs a matrix with the given shape, each cell initialized with zero.
 */
public DoubleMatrix2D make(int rows, int columns) {
	if (this==sparse) return new SparseDoubleMatrix2D(rows,columns);
	if (this==rowCompressed) return new RCDoubleMatrix2D(rows,columns);
	//if (this==rowCompressedModified) return new RCMDoubleMatrix2D(rows,columns);
	else return new DenseDoubleMatrix2D(rows,columns);
}
 

/**
 * Constructs a matrix with the given shape, each cell initialized with zero.
 */
public DoubleMatrix2D make(int rows, int columns) {
	if (this==sparse) return new SparseDoubleMatrix2D(rows,columns);
	if (this==rowCompressed) return new RCDoubleMatrix2D(rows,columns);
	//if (this==rowCompressedModified) return new RCMDoubleMatrix2D(rows,columns);
	else return new DenseDoubleMatrix2D(rows,columns);
}
 

private void checkFullRank(DesignMatrix designMatrix) throws XMLParseException {
        int fullRank = designMatrix.getColumnDimension();
//        System.err.println("designMatrix getColumnDimension = "+fullRank);
        SingularValueDecomposition svd = new SingularValueDecomposition(
                new DenseDoubleMatrix2D(designMatrix.getParameterAsMatrix()));
        int realRank = svd.rank();
        if (realRank != fullRank) {
            throw new XMLParseException(
                "rank(" + designMatrix.getId() + ") = " + realRank +
                        ".\nMatrix is not of full rank as colDim(" + designMatrix.getId() + ") = " + fullRank        
            );
        }
    }
 

private void checkFullRank(DesignMatrix designMatrix) throws XMLParseException {
        int fullRank = designMatrix.getColumnDimension();
//        System.err.println("designMatrix getColumnDimension = "+fullRank);
        SingularValueDecomposition svd = new SingularValueDecomposition(
                new DenseDoubleMatrix2D(designMatrix.getParameterAsMatrix()));
        int realRank = svd.rank();
        if (realRank != fullRank) {
            throw new XMLParseException(
                    "rank(" + designMatrix.getId() + ") = " + realRank +
                            ".\nMatrix is not of full rank as colDim(" + designMatrix.getId() + ") = " + fullRank
            );
        }
    }
 

public MultivariateNormalOperator(Parameter parameter, double scaleFactor, double[][] inMatrix, double weight,
                                  AdaptationMode mode, boolean isVarianceMatrix) {

    super(mode);
    this.scaleFactor = scaleFactor;
    this.parameter = parameter;
    setWeight(weight);
    dim = parameter.getDimension();

    SingularValueDecomposition svd = new SingularValueDecomposition(new DenseDoubleMatrix2D(inMatrix));
    if (inMatrix[0].length != svd.rank()) {
        throw new RuntimeException("Variance matrix in mvnOperator is not of full rank");
    }

    final double[][] matrix;
    if (isVarianceMatrix) {
        matrix = inMatrix;
    } else {
        matrix = formXtXInverse(inMatrix);
    }

    try {
        cholesky = (new CholeskyDecomposition(matrix)).getL();
    } catch (IllegalDimension illegalDimension) {
        throw new RuntimeException("Unable to decompose matrix in mvnOperator");
    }
}
 

public ModeIndependenceOperator(Parameter parameter, double scaleFactor, double[][] inMatrix, double weight,
                                    AdaptationMode mode, boolean isVarianceMatrix) {

        super(mode);
        this.scaleFactor = scaleFactor;
        this.parameter = parameter;
        setWeight(weight);
        dim = parameter.getDimension();

        SingularValueDecomposition svd = new SingularValueDecomposition(new DenseDoubleMatrix2D(inMatrix));
        if (inMatrix[0].length != svd.rank()) {
            throw new RuntimeException("Variance matrix in mvnOperator is not of full rank");
        }

        final double[][] matrix;
        if (isVarianceMatrix) {
            matrix = inMatrix;
        } else {
            matrix = formXtXInverse(inMatrix);
        }

//        System.err.println("Matrix:");
//        System.err.println(new Matrix(matrix));

        try {
            cholesky = (new CholeskyDecomposition(matrix)).getL();
        } catch (IllegalDimension illegalDimension) {
            throw new RuntimeException("Unable to decompose matrix in mvnOperator");
        }

//        System.err.println("Cholesky:");
//        System.err.println(new Matrix(cholesky));
//        System.exit(-1);
    }
 

@Override
public double error(Factorization<U, I> factorization, FastPreferenceData<U, I> data) {
    DenseDoubleMatrix2D pu_z = factorization.getUserMatrix();
    DenseDoubleMatrix2D piz = factorization.getItemMatrix();

    return data.getUidxWithPreferences().parallel().mapToDouble(uidx -> {
        DoubleMatrix1D pU_z = pu_z.viewRow(uidx);
        DoubleMatrix1D pUi = piz.zMult(pU_z, null);
        return data.getUidxPreferences(uidx)
                .mapToDouble(iv -> -iv.v2 * pUi.getQuick(iv.v1))
                .sum();
    }).sum();

}
 

private static Matrix getMatrixSqrt(Matrix M, Boolean invert) {
    DoubleMatrix2D S = new DenseDoubleMatrix2D(M.toComponents());
    RobustEigenDecomposition eigenDecomp = new RobustEigenDecomposition(S, 100);
    DoubleMatrix1D eigenValues = eigenDecomp.getRealEigenvalues();
    int dim = eigenValues.size();
    for (int i = 0; i < dim; i++) {
        double value = sqrt(eigenValues.get(i));
        if (invert) {
            value = 1 / value;
        }
        eigenValues.set(i, value);
    }

    DoubleMatrix2D eigenVectors = eigenDecomp.getV();
    for (int i = 0; i < dim; i++) {
        for (int j = 0; j < dim; j++) {
            eigenVectors.set(i, j, eigenVectors.get(i, j) * eigenValues.get(j));

        }
    }
    DoubleMatrix2D storageMatrix = new DenseDoubleMatrix2D(dim, dim);
    eigenVectors.zMult(eigenDecomp.getV(), storageMatrix, 1, 0, false, true);


    return new Matrix(storageMatrix.toArray());

}
 

private static void saveDenseDoubleMatrix2D(OutputStream stream, DenseDoubleMatrix2D matrix) throws IOException {
    BufferedWriter out = new BufferedWriter(new OutputStreamWriter(stream));
    double[][] m = matrix.toArray();
    for (double[] pu : m) {
        for (int j = 0; j < pu.length - 1; j++) {
            out.write(Double.toString(pu[j]));
            out.write('\t');
        }
        out.write(Double.toString(pu[pu.length - 1]));
        out.newLine();
    }
    out.flush();
}
 

/**
 * Expectation step of the EM algorithm.
 *
 * @param pz_u   matrix of p(z|u)
 * @param piz    matrix of p(i|z)
 * @param qzData PLSA preference data (variational probability Q(z))
 */
protected void expectation(final DenseDoubleMatrix2D pz_u, final DenseDoubleMatrix2D piz, PLSAPreferenceData<U, I> qzData) {
    qzData.getUidxWithPreferences().parallel()
            .forEach(uidx -> qzData.getUidxPreferences(uidx)
                    .forEach(iqz -> {
                        int iidx = iqz.v1;
                        double[] qz = ((PLSAPreferenceData.PLSAIdxPref) iqz).qz;
                        for (int z = 0; z < qz.length; z++) {
                            qz[z] = piz.getQuick(iidx, z) * pz_u.getQuick(uidx, z);
                        }
                        normalizeQz(qz);
                    }));
}
 

@Override
public double error(Factorization<U, I> factorization, FastPreferenceData<U, I> data) {

    DenseDoubleMatrix2D p = factorization.getUserMatrix();
    DenseDoubleMatrix2D q = factorization.getItemMatrix();

    return error(p, q, data);
}
 

/**
 * Constructor.
 *
 * @param uIndex fast user index
 * @param iIndex fast item index
 * @param K dimension of the latent feature space
 * @param initFunction function to initialize the cells of the matrices
 */
public Factorization(FastUserIndex<U> uIndex, FastItemIndex<I> iIndex, int K, DoubleFunction initFunction) {
    this.userMatrix = new DenseDoubleMatrix2D(uIndex.numUsers(), K);
    this.userMatrix.assign(initFunction);
    this.itemMatrix = new DenseDoubleMatrix2D(iIndex.numItems(), K);
    this.itemMatrix.assign(initFunction);
    this.K = K;
    this.uIndex = uIndex;
    this.iIndex = iIndex;
}
 

private static DenseDoubleMatrix2D loadDenseDoubleMatrix2D(InputStream stream, int rows, int columns) throws IOException {
    double[][] m = new double[rows][columns];

    BufferedReader in = new BufferedReader(new InputStreamReader(stream));
    for (double[] mi : m) {
        CharSequence[] tokens = split(in.readLine(), '\t', mi.length);
        for (int j = 0; j < mi.length; j++) {
            mi[j] = parseDouble(tokens[j].toString());
        }
    }

    return new DenseDoubleMatrix2D(m);
}
 

/**
 * Constructor that initialises some mock data.
 */
public MFRecommenderTest() {
    FastUserIndex<String> uIndex = new SimpleFastUserIndex<String>() {
        {
            add("0");
        }
    };
    FastItemIndex<String> iIndex = new SimpleFastItemIndex<String>() {
        {
            add("0");
            add("1");
            add("2");
            add("3");
            add("4");
            add("5");
        }
    };

    int K = 2;
    DenseDoubleMatrix2D p = new DenseDoubleMatrix2D(new double[][]{
        new double[]{0.1, 1.0}
    });
    DenseDoubleMatrix2D q = new DenseDoubleMatrix2D(new double[][]{
        new double[]{1.0, 6.0},
        new double[]{1.0, 5.0},
        new double[]{1.0, 4.0},
        new double[]{1.0, 3.0},
        new double[]{1.0, 2.0},
        new double[]{1.0, 1.0}
    });
    Factorization<String, String> factorization = new Factorization<String, String>(uIndex, iIndex, p, q, K) {
    };

    recommender = new MFRecommender<>(uIndex, iIndex, factorization);
}
 

@Override
public void set_minP(final DenseDoubleMatrix2D p, final DenseDoubleMatrix2D q, FastPreferenceData<U, I> data) {
    set_min(p, q, confidence, lambdaP, data);
}
 

/**
 * Calculates based on windows of magnitude spectrum. Encompasses portion of
 * Moments class, but has a delay of lengthOfWindow windows before any
 * results are calculated.
 * 
 * @param samples
 *            The samples to extract the feature from.
 * @param sampling_rate
 *            The sampling rate that the samples are encoded with.
 * @param other_feature_values
 *            The values of other features that are needed to calculate this
 *            value. The order and offsets of these features must be the
 *            same as those returned by this class's getDependencies and
 *            getDependencyOffsets methods respectively. The first indice
 *            indicates the feature/window and the second indicates the
 *            value.
 * @return The extracted feature value(s).
 * @throws Exception
 *             Throws an informative exception if the feature cannot be
 *             calculated.
 */
public double[] extractFeature(double[] samples, double sampling_rate,
		double[][] other_feature_values) throws Exception {
	if((featureLength != other_feature_values[0].length)||(windowLength != other_feature_values.length)){
		terms = new DenseDoubleMatrix2D(k*l,windowLength*featureLength);
		z = new DenseDoubleMatrix2D(1,featureLength*windowLength);
		calcTerms(terms);
	}
	for(int i=0;i<windowLength;++i){
		for(int j=0;j<featureLength;++j){
			z.set(0,featureLength*i+j,other_feature_values[i][j]);
		}
	}
	DoubleMatrix2D retMatrix = (new Algebra()).solve(terms,z);
	return retMatrix.viewRow(0).toArray();
}
 

/**
 * Calculates based on windows of magnitude spectrum. Encompasses portion of
 * Moments class, but has a delay of lengthOfWindow windows before any
 * results are calculated.
 * 
 * @param samples
 *            The samples to extract the feature from.
 * @param sampling_rate
 *            The sampling rate that the samples are encoded with.
 * @param other_feature_values
 *            The values of other features that are needed to calculate this
 *            value. The order and offsets of these features must be the
 *            same as those returned by this class's getDependencies and
 *            getDependencyOffsets methods respectively. The first indice
 *            indicates the feature/window and the second indicates the
 *            value.
 * @return The extracted feature value(s).
 * @throws Exception
 *             Throws an informative exception if the feature cannot be
 *             calculated.
 */
public double[] extractFeature(double[] samples, double sampling_rate,
		double[][] other_feature_values) throws Exception {
	if((featureLength != other_feature_values[0].length)||(windowLength != other_feature_values.length)){
		terms = new DenseDoubleMatrix2D(k*l,windowLength*featureLength);
		z = new DenseDoubleMatrix2D(1,featureLength*windowLength);
		calcTerms(terms);
	}
	for(int i=0;i<windowLength;++i){
		for(int j=0;j<featureLength;++j){
			z.set(0,featureLength*i+j,other_feature_values[i][j]);
		}
	}
	DoubleMatrix2D retMatrix = (new Algebra()).solve(terms,z);
	return retMatrix.viewRow(0).toArray();
}