org.apache.commons.math3.stat.ranking.NaturalRanking Java Examples

/**
 * Computes the Spearman's rank correlation coefficient between the two arrays.
 *
 * @param xArray first data array
 * @param yArray second data array
 * @return Returns Spearman's rank correlation coefficient for the two arrays
 * @throws DimensionMismatchException if the arrays lengths do not match
 * @throws MathIllegalArgumentException if the array length is less than 2
 */
public double correlation(final double[] xArray, final double[] yArray) {
    if (xArray.length != yArray.length) {
        throw new DimensionMismatchException(xArray.length, yArray.length);
    } else if (xArray.length < 2) {
        throw new MathIllegalArgumentException(LocalizedFormats.INSUFFICIENT_DIMENSION,
                                               xArray.length, 2);
    } else {
        double[] x = xArray;
        double[] y = yArray;
        if (rankingAlgorithm instanceof NaturalRanking &&
            NaNStrategy.REMOVED == ((NaturalRanking) rankingAlgorithm).getNanStrategy()) {
            final Set<Integer> nanPositions = new HashSet<Integer>();

            nanPositions.addAll(getNaNPositions(xArray));
            nanPositions.addAll(getNaNPositions(yArray));

            x = removeValues(xArray, nanPositions);
            y = removeValues(yArray, nanPositions);
        }
        return new PearsonsCorrelation().correlation(rankingAlgorithm.rank(x), rankingAlgorithm.rank(y));
    }
}
 

@Test()
public void testRankOfRows() {
    final Random random = new Random();
    final DataFrame<String,String> frame = TestDataFrames.random(double.class, 1000, 100);
    frame.applyDoubles(v -> random.nextDouble() * 100);
    final DataFrame<String,String> rankFrame = frame.rank().ofRows();
    Assert.assertEquals(rankFrame.rowCount(), frame.rowCount(), "The row counts match");
    Assert.assertEquals(rankFrame.colCount(), frame.colCount(), "The column counts match");
    rankFrame.out().print();
    rankFrame.rows().forEach(rankRow -> {
        final String key = rankRow.key();
        final double[] values = frame.row(key).toDoubleStream().toArray();
        final NaturalRanking ranking = new NaturalRanking();
        final double[] ranks = ranking.rank(values);
        for (int i = 0; i < ranks.length; ++i) {
            final double value = frame.data().getDouble(key, i);
            final double expected = ranks[i];
            final double actual = rankFrame.data().getDouble(key, i);
            Assert.assertEquals(value, values[i], "The values match for column " + i);
            Assert.assertEquals(actual, expected, "The ranks match for " + key + " at column " + i);
        }
    });
}
 

@Test()
public void testRankOfColumns() {
    final Random random = new Random();
    final DataFrame<String,String> frame = TestDataFrames.random(double.class, 1000, 100);
    frame.applyDoubles(v -> random.nextDouble() * 100);
    final DataFrame<String,String> rankFrame = frame.rank().ofColumns();
    Assert.assertEquals(rankFrame.rowCount(), frame.rowCount(), "The row counts match");
    Assert.assertEquals(rankFrame.colCount(), frame.colCount(), "The column counts match");
    rankFrame.out().print();
    rankFrame.cols().forEach(rankColumn -> {
        final String key = rankColumn.key();
        final double[] values = frame.col(key).toDoubleStream().toArray();
        final NaturalRanking ranking = new NaturalRanking();
        final double[] ranks = ranking.rank(values);
        for (int i=0; i<ranks.length; ++i) {
            final double value = frame.data().getDouble(i, key);
            final double expected = ranks[i];
            final double actual = rankFrame.data().getDouble(i, key);
            Assert.assertEquals(value, values[i], "The values match for row " + i);
            Assert.assertEquals(actual, expected, "The ranks match for " + key + " at row " + i);
        }
    });
}
 

/**
 * Computes the Spearman's rank correlation coefficient between the two arrays.
 *
 * @param xArray first data array
 * @param yArray second data array
 * @return Returns Spearman's rank correlation coefficient for the two arrays
 * @throws DimensionMismatchException if the arrays lengths do not match
 * @throws MathIllegalArgumentException if the array length is less than 2
 */
public double correlation(final double[] xArray, final double[] yArray) {
    if (xArray.length != yArray.length) {
        throw new DimensionMismatchException(xArray.length, yArray.length);
    } else if (xArray.length < 2) {
        throw new MathIllegalArgumentException(LocalizedFormats.INSUFFICIENT_DIMENSION,
                                               xArray.length, 2);
    } else {
        double[] x = xArray;
        double[] y = yArray;
        if (rankingAlgorithm instanceof NaturalRanking &&
            NaNStrategy.REMOVED == ((NaturalRanking) rankingAlgorithm).getNanStrategy()) {
            final Set<Integer> nanPositions = new HashSet<Integer>();

            nanPositions.addAll(getNaNPositions(xArray));
            nanPositions.addAll(getNaNPositions(yArray));

            x = removeValues(xArray, nanPositions);
            y = removeValues(yArray, nanPositions);
        }
        return new PearsonsCorrelation().correlation(rankingAlgorithm.rank(x), rankingAlgorithm.rank(y));
    }
}
 

@Test
public void testMath891Matrix() {
    final double[] xArray = new double[] { Double.NaN, 1.9, 2, 100, 3 };
    final double[] yArray = new double[] { 10, 2, 10, Double.NaN, 4 };

    RealMatrix matrix = MatrixUtils.createRealMatrix(xArray.length, 2);
    for (int i = 0; i < xArray.length; i++) {
        matrix.addToEntry(i, 0, xArray[i]);
        matrix.addToEntry(i, 1, yArray[i]);
    }

    // compute correlation
    NaturalRanking ranking = new NaturalRanking(NaNStrategy.REMOVED);
    SpearmansCorrelation spearman = new SpearmansCorrelation(matrix, ranking);
    
    Assert.assertEquals(0.5, spearman.getCorrelationMatrix().getEntry(0, 1), Double.MIN_VALUE);
}
 

@Test
public void testMath891Matrix() {
    final double[] xArray = new double[] { Double.NaN, 1.9, 2, 100, 3 };
    final double[] yArray = new double[] { 10, 2, 10, Double.NaN, 4 };

    RealMatrix matrix = MatrixUtils.createRealMatrix(xArray.length, 2);
    for (int i = 0; i < xArray.length; i++) {
        matrix.addToEntry(i, 0, xArray[i]);
        matrix.addToEntry(i, 1, yArray[i]);
    }

    // compute correlation
    NaturalRanking ranking = new NaturalRanking(NaNStrategy.REMOVED);
    SpearmansCorrelation spearman = new SpearmansCorrelation(matrix, ranking);
    
    Assert.assertEquals(0.5, spearman.getCorrelationMatrix().getEntry(0, 1), Double.MIN_VALUE);
}
 

private void forceNormalExpressionData(ExpressionDataset datasetExpression) throws ArithmeticException {
	System.out.println("Enforcing normal distribution on expression data:");

	NaturalRanking ranker = new NaturalRanking();

	for (int p = 0; p < datasetExpression.nrProbes; p++) {
		//Rank order the expression values:
		double[] values = new double[datasetExpression.nrSamples];
		for (int s = 0; s < datasetExpression.nrSamples; s++) {
			values[s] = datasetExpression.rawData[p][s];
		}

		double[] rankedValues = ranker.rank(values);
		//Replace the original expression value with the standard distribution enforce:
		for (int s = 0; s < datasetExpression.nrSamples; s++) {
			//Convert the rank to a proportion, with range <0, 1>
			double pValue = (0.5d + rankedValues[s] - 1d) / (double) (rankedValues.length);
			//Convert the pValue to a Z-Score:
			double zScore = cern.jet.stat.tdouble.Probability.normalInverse(pValue);
			datasetExpression.rawData[p][s] = zScore; //Replace original expression value with the Z-Score
		}
	}

	System.out.println("Expression data now force normal");
}
 

/**
 * Computes the Spearman's rank correlation coefficient between the two arrays.
 *
 * @param xArray first data array
 * @param yArray second data array
 * @return Returns Spearman's rank correlation coefficient for the two arrays
 * @throws DimensionMismatchException if the arrays lengths do not match
 * @throws MathIllegalArgumentException if the array length is less than 2
 */
public double correlation(final double[] xArray, final double[] yArray) {
    if (xArray.length != yArray.length) {
        throw new DimensionMismatchException(xArray.length, yArray.length);
    } else if (xArray.length < 2) {
        throw new MathIllegalArgumentException(LocalizedFormats.INSUFFICIENT_DIMENSION,
                                               xArray.length, 2);
    } else {
        double[] x = xArray;
        double[] y = yArray;
        if (rankingAlgorithm instanceof NaturalRanking &&
            NaNStrategy.REMOVED == ((NaturalRanking) rankingAlgorithm).getNanStrategy()) {
            final Set<Integer> nanPositions = new HashSet<Integer>();

            nanPositions.addAll(getNaNPositions(xArray));
            nanPositions.addAll(getNaNPositions(yArray));

            x = removeValues(xArray, nanPositions);
            y = removeValues(yArray, nanPositions);
        }
        return new PearsonsCorrelation().correlation(rankingAlgorithm.rank(x), rankingAlgorithm.rank(y));
    }
}
 

private void forceNormalCovariates(ExpressionDataset datasetCovariates, ExpressionDataset datasetGenotypes) throws ArithmeticException {
	System.out.println("Enforcing normal distribution on covariates");

	NaturalRanking ranker = new NaturalRanking();

	for (int p = 0; p < datasetCovariates.nrProbes; p++) {
		//Rank order the expression values:
		double[] values = new double[datasetCovariates.nrSamples];
		for (int s = 0; s < datasetGenotypes.nrSamples; s++) {
			values[s] = datasetCovariates.rawData[p][s];
		}
		double[] rankedValues = ranker.rank(values);
		//Replace the original expression value with the standard distribution enforce:
		for (int s = 0; s < datasetGenotypes.nrSamples; s++) {
			//Convert the rank to a proportion, with range <0, 1>
			double pValue = (0.5d + rankedValues[s] - 1d) / (double) (rankedValues.length);
			//Convert the pValue to a Z-Score:
			double zScore = cern.jet.stat.tdouble.Probability.normalInverse(pValue);
			datasetCovariates.rawData[p][s] = zScore; //Replace original expression value with the Z-Score
		}
	}
}
 

@Test
public void testMath891Matrix() {
    final double[] xArray = new double[] { Double.NaN, 1.9, 2, 100, 3 };
    final double[] yArray = new double[] { 10, 2, 10, Double.NaN, 4 };

    RealMatrix matrix = MatrixUtils.createRealMatrix(xArray.length, 2);
    for (int i = 0; i < xArray.length; i++) {
        matrix.addToEntry(i, 0, xArray[i]);
        matrix.addToEntry(i, 1, yArray[i]);
    }

    // compute correlation
    NaturalRanking ranking = new NaturalRanking(NaNStrategy.REMOVED);
    SpearmansCorrelation spearman = new SpearmansCorrelation(matrix, ranking);
    
    Assert.assertEquals(0.5, spearman.getCorrelationMatrix().getEntry(0, 1), Double.MIN_VALUE);
}
 

/**
 * Computes the Spearman's rank correlation coefficient between the two arrays.
 *
 * @param xArray first data array
 * @param yArray second data array
 * @return Returns Spearman's rank correlation coefficient for the two arrays
 * @throws DimensionMismatchException if the arrays lengths do not match
 * @throws MathIllegalArgumentException if the array length is less than 2
 */
public double correlation(final double[] xArray, final double[] yArray) {
    if (xArray.length != yArray.length) {
        throw new DimensionMismatchException(xArray.length, yArray.length);
    } else if (xArray.length < 2) {
        throw new MathIllegalArgumentException(LocalizedFormats.INSUFFICIENT_DIMENSION,
                                               xArray.length, 2);
    } else {
        double[] x = xArray;
        double[] y = yArray;
        if (rankingAlgorithm instanceof NaturalRanking &&
            NaNStrategy.REMOVED == ((NaturalRanking) rankingAlgorithm).getNanStrategy()) {
            final Set<Integer> nanPositions = new HashSet<Integer>();

            nanPositions.addAll(getNaNPositions(xArray));
            nanPositions.addAll(getNaNPositions(yArray));

            x = removeValues(xArray, nanPositions);
            y = removeValues(yArray, nanPositions);
        }
        return new PearsonsCorrelation().correlation(rankingAlgorithm.rank(x), rankingAlgorithm.rank(y));
    }
}
 

/**
 * Returns the rank array for the values specified
 * @param values    the values to rank
 * @return          the ranks of input array
 */
static double[] rank(double[] values) {
    final NaNStrategy nanStrategy = (NaNStrategy)optionsMap.get(NaNStrategy.class).get(DataFrameOptions.getNanStrategy());
    final TiesStrategy tieStrategy = (TiesStrategy)optionsMap.get(TiesStrategy.class).get(DataFrameOptions.getTieStrategy());
    if (nanStrategy == null) throw new DataFrameException("Unsupported NaN strategy specified: " + DataFrameOptions.getNanStrategy());
    if (tieStrategy == null) throw new DataFrameException("Unsupported tie strategy specified: " + DataFrameOptions.getTieStrategy());
    final NaturalRanking ranking = new NaturalRanking(nanStrategy, tieStrategy);
    return ranking.rank(values);
}
 

@Test
public void testMath891Array() {
    final double[] xArray = new double[] { Double.NaN, 1.9, 2, 100, 3 };
    final double[] yArray = new double[] { 10, 2, 10, Double.NaN, 4 };

    NaturalRanking ranking = new NaturalRanking(NaNStrategy.REMOVED);
    SpearmansCorrelation spearman = new SpearmansCorrelation(ranking);
    
    Assert.assertEquals(0.5, spearman.correlation(xArray, yArray), Double.MIN_VALUE);
}
 

@Test
public void testMath891Array() {
    final double[] xArray = new double[] { Double.NaN, 1.9, 2, 100, 3 };
    final double[] yArray = new double[] { 10, 2, 10, Double.NaN, 4 };

    NaturalRanking ranking = new NaturalRanking(NaNStrategy.REMOVED);
    SpearmansCorrelation spearman = new SpearmansCorrelation(ranking);
    
    Assert.assertEquals(0.5, spearman.correlation(xArray, yArray), Double.MIN_VALUE);
}
 

@Test
public void testMath891Array() {
    final double[] xArray = new double[] { Double.NaN, 1.9, 2, 100, 3 };
    final double[] yArray = new double[] { 10, 2, 10, Double.NaN, 4 };

    NaturalRanking ranking = new NaturalRanking(NaNStrategy.REMOVED);
    SpearmansCorrelation spearman = new SpearmansCorrelation(ranking);
    
    Assert.assertEquals(0.5, spearman.correlation(xArray, yArray), Double.MIN_VALUE);
}
 

@Test
public void testMath891Array() {
    final double[] xArray = new double[] { Double.NaN, 1.9, 2, 100, 3 };
    final double[] yArray = new double[] { 10, 2, 10, Double.NaN, 4 };

    NaturalRanking ranking = new NaturalRanking(NaNStrategy.REMOVED);
    SpearmansCorrelation spearman = new SpearmansCorrelation(ranking);
    
    Assert.assertEquals(0.5, spearman.correlation(xArray, yArray), Double.MIN_VALUE);
}
 

public static void rankColumns(DoubleMatrix2D matrix) {
    RankingAlgorithm COV_RANKER_TIE = new NaturalRanking(NaNStrategy.FAILED, TiesStrategy.AVERAGE);
    
    for (int c = 0; c < matrix.columns(); c++) {
        double[] rank = COV_RANKER_TIE.rank(matrix.viewColumn(c).toArray());
        for (int r = 0; r < matrix.rows(); r++) {
            matrix.set(r, c, (rank[r]));
        }
    }
}
 

@Override
public Object doWork(Object value){
  if(null == value){
    return null;
  }
  else if(value instanceof List){
    NaturalRanking rank = new NaturalRanking();      
    return Arrays.stream(rank.rank(((List<?>)value).stream().mapToDouble(innerValue -> ((Number)innerValue).doubleValue()).toArray())).boxed().collect(Collectors.toList());
  }
  else{
    return doWork(Arrays.asList((Number)value));
  }
}
 

/**
 * Create a SpearmansCorrelation without data.
 */
public SpearmansCorrelation() {
    this(new NaturalRanking());
}
 

private ExpressionDataset correctCovariateDataPCA(String[] covsToCorrect2, String[] covsToCorrect, ExpressionDataset datasetGenotypes, ExpressionDataset datasetCovariatesPCAForceNormal, int nrCompsToCorrectFor) throws Exception {

		System.out.println("Preparing data for testing eQTL effects of SNPs on covariate data:");
		System.out.println("Correcting covariate data for cohort specific effects:");

		ExpressionDataset datasetCovariatesToCorrectFor = new ExpressionDataset(covsToCorrect2.length + covsToCorrect.length + nrCompsToCorrectFor, datasetGenotypes.nrSamples);
		datasetCovariatesToCorrectFor.sampleNames = datasetGenotypes.sampleNames;

		// add covariates from the first list
		HashMap hashCovsToCorrect = new HashMap();

		// add covariates from the second list
		for (int i = 0; i < covsToCorrect2.length; ++i) {
			String cov = covsToCorrect2[i];
			hashCovsToCorrect.put(cov, null);
			Integer c = datasetCovariatesPCAForceNormal.hashProbes.get(cov);
			if (c == null) {
				throw new Exception("Covariate not found: " + cov);
			}
			for (int s = 0; s < datasetGenotypes.nrSamples; s++) {
				datasetCovariatesToCorrectFor.rawData[i][s] = datasetCovariatesPCAForceNormal.rawData[c][s];
			}
		}

		int[] covsToCorrectIndex = new int[covsToCorrect.length];
		for (int c = 0; c < covsToCorrect.length; c++) {
			hashCovsToCorrect.put(covsToCorrect[c], null);
			covsToCorrectIndex[c] = ((Integer) datasetCovariatesPCAForceNormal.hashProbes.get(covsToCorrect[c])).intValue();
			for (int s = 0; s < datasetGenotypes.nrSamples; s++) {
				datasetCovariatesToCorrectFor.rawData[covsToCorrect2.length + c][s] = datasetCovariatesPCAForceNormal.rawData[covsToCorrectIndex[c]][s];
			}
		}

		// add PCs
		if (nrCompsToCorrectFor > 0) {
			for (int comp = 0; comp < nrCompsToCorrectFor; comp++) {
				for (int s = 0; s < datasetGenotypes.nrSamples; s++) {
					datasetCovariatesToCorrectFor.rawData[covsToCorrect2.length + covsToCorrect.length + comp][s] = datasetCovariatesPCAForceNormal.rawData[datasetCovariatesPCAForceNormal.nrProbes - 51 + comp][s];
				}
			}
		}

		datasetCovariatesToCorrectFor.transposeDataset();

		datasetCovariatesToCorrectFor.save(inputDir + "/CovariatesToCorrectFor.txt");
		orthogonalizeDataset(inputDir + "/CovariatesToCorrectFor.txt");
		datasetCovariatesToCorrectFor = new ExpressionDataset(inputDir + "/CovariatesToCorrectFor.txt.PrincipalComponents.txt");
		datasetCovariatesToCorrectFor.transposeDataset();
		ExpressionDataset datasetCovariatesToCorrectForEigenvalues = new ExpressionDataset(inputDir + "/CovariatesToCorrectFor.txt.Eigenvalues.txt");

		for (int p = 0; p < datasetCovariatesPCAForceNormal.nrProbes; p++) {
			if (!hashCovsToCorrect.containsKey(datasetCovariatesPCAForceNormal.probeNames[p])) {
				for (int cov = 0; cov < datasetCovariatesToCorrectFor.nrProbes; cov++) {
					if (datasetCovariatesToCorrectForEigenvalues.rawData[cov][0] > 1E-5) {
						double[] rc = getLinearRegressionCoefficients(datasetCovariatesToCorrectFor.rawData[cov], datasetCovariatesPCAForceNormal.rawData[p]);
						for (int s = 0; s < datasetGenotypes.nrSamples; s++) {
							datasetCovariatesPCAForceNormal.rawData[p][s] -= rc[0] * datasetCovariatesToCorrectFor.rawData[cov][s];
						}
					}
				}
				/*double stdev = JSci.maths.ArrayMath.standardDeviation(datasetCovariates.rawData[p]);
				 double mean = JSci.maths.ArrayMath.mean(datasetCovariates.rawData[p]);
				 if (stdev < 1E-5) {
				 for (int s = 0; s < datasetGenotypes.nrSamples; s++) {
				 datasetCovariatesPCAForceNormal.rawData[p][s] = mean;
				 }
				 }*/
			}
		}

		System.out.println("Enforcing normal distribution on covariates");

		NaturalRanking ranker = new NaturalRanking();

		for (int p = 0; p < datasetCovariatesPCAForceNormal.nrProbes; p++) {
			//Rank order the expression values:
			double[] values = new double[datasetCovariatesPCAForceNormal.nrSamples];
			for (int s = 0; s < datasetGenotypes.nrSamples; s++) {
				values[s] = datasetCovariatesPCAForceNormal.rawData[p][s];
			}
			double[] rankedValues = ranker.rank(values);
			//Replace the original expression value with the standard distribution enforce:
			for (int s = 0; s < datasetGenotypes.nrSamples; s++) {
				//Convert the rank to a proportion, with range <0, 1>
				double pValue = (0.5d + rankedValues[s] - 1d) / (double) (rankedValues.length);
				//Convert the pValue to a Z-Score:
				double zScore = cern.jet.stat.tdouble.Probability.normalInverse(pValue);
				datasetCovariatesPCAForceNormal.rawData[p][s] = zScore; //Replace original expression value with the Z-Score
			}
		}
		return datasetCovariatesPCAForceNormal;
	}
 

/**
 * Create a SpearmansCorrelation without data.
 */
public SpearmansCorrelation() {
    data = null;
    this.rankingAlgorithm = new NaturalRanking();
    rankCorrelation = null;
}
 

public MannWhitneyUTest2() {
	naturalRanking = new NaturalRanking(NaNStrategy.FIXED,
			TiesStrategy.AVERAGE);
}
 

/**
 * Create a SpearmansCorrelation without data.
 */
public SpearmansCorrelation() {
    data = null;
    this.rankingAlgorithm = new NaturalRanking();
    rankCorrelation = null;
}
 

/**
 * Create a SpearmansCorrelation without data.
 */
public SpearmansCorrelation() {
    data = null;
    this.rankingAlgorithm = new NaturalRanking();
    rankCorrelation = null;
}
 

public static double[] toRanks(double[] x) {
	NaturalRanking ranking = new NaturalRanking();
	return ranking.rank(x);
}
 

/**
 * Create a SpearmansCorrelation without data.
 */
public SpearmansCorrelation() {
    this(new NaturalRanking());
}
 

/**
 * Create a SpearmansCorrelation without data.
 */
public SpearmansCorrelation() {
    this(new NaturalRanking());
}
 

/**
 * Create a SpearmansCorrelation without data.
 */
public SpearmansCorrelation() {
    this(new NaturalRanking());
}
 

/**
 * Create a SpearmansCorrelation without data.
 */
public SpearmansCorrelation() {
    this(new NaturalRanking());
}
 

public NaturalRanking getNaturalRanking() {
	return naturalRanking;
}