org.apache.commons.math3.stat.correlation.PearsonsCorrelation Java Examples

/**
 * Calculates pearson correlation between pair of columns in the in the matrix, assuming that there is a strict
 * one to one relationship between the matrix columns and the field specifications in the list.
 *
 * Writes the correlation, pValue and standard error to a file using JSON format.
 *
 * @param matrix the input matrix where fields are represented by as columns and subjects by rows
 * @param fields a list of field specifications for which the correlations are to be calculated
 * @param correlationAnalysisOutputPath is the file to which the results are written
 * @throws Exception
 */
public static void calculateAndOutputCorrelations(RealMatrix matrix, List<FieldRecipe> fields,
                                                   String correlationAnalysisOutputPath) throws Exception {
    PearsonsCorrelation correlation = new PearsonsCorrelation(matrix);
    RealMatrix correlationMatrix = correlation.getCorrelationMatrix();
    RealMatrix pValueMatrix = correlation.getCorrelationPValues();
    RealMatrix standardErrorMatrix = correlation.getCorrelationStandardErrors();

    // Output the correlation analysis
    JSONArray correlationArray = new JSONArray();
    for (int i=0; i<correlationMatrix.getRowDimension(); i++){
        for (int j=0; j<correlationMatrix.getColumnDimension(); j++){
            JSONObject correlationObject = new JSONObject();
            correlationObject.put("xFieldLabel", fields.get(i).toField().getLabel());
            correlationObject.put("yFieldLabel", fields.get(j).toField().getLabel());
            correlationObject.put("correlationCoefficient", correlationMatrix.getEntry(i,j));
            correlationObject.put("pValue", pValueMatrix.getEntry(i,j));
            correlationObject.put("standardError", standardErrorMatrix.getEntry(i,j));
            correlationArray.add(correlationObject);
        }
    }
    Writer writer = new OutputStreamWriter(new FileOutputStream(correlationAnalysisOutputPath), "UTF-8");
    writer.write(correlationArray.toJSONString());
    writer.flush();
    writer.close();
}
 

@Override
public Object doWork(Object value) throws IOException{
  if(null == value){
    return null;
  } else if(value instanceof Matrix) {
    Matrix matrix = (Matrix) value;
    Object corr = matrix.getAttribute("corr");
    if(corr instanceof PearsonsCorrelation) {
      PearsonsCorrelation pcorr = (PearsonsCorrelation)corr;
      RealMatrix  realMatrix = pcorr.getCorrelationPValues();
      return new Matrix(realMatrix.getData());
    } else {
      throw new IOException("Correlation pvalues are only available for Pearsons and Spearmans correlations");
    }
  } else {
    throw new IOException("matrix parameter expected for transpose function");
  }
}
 

@Override
protected Object getExpectedValue(int start, int length)
{
    if (length <= 1) {
        return null;
    }
    PearsonsCorrelation corr = new PearsonsCorrelation();
    return corr.correlation(constructDoublePrimitiveArray(start + 2, length), constructDoublePrimitiveArray(start, length));
}
 

public static double getPearson(List<Double> list1, List<Double> list2)
{
	PearsonsCorrelation correlation = new PearsonsCorrelation();
	double c = correlation.correlation(getArray(list1),getArray(list2));
		
	return c;
}
 

/**
 * @return Pearson's product-moment correlation coefficients for the measured data
 */
private double[] computePearson() {
    if(freqLeqStats.size() < 3) {
        return null;
    }
    // Frequency count, one dataset by frequency
    int dataSetCount = freqLeqStats.get(freqLeqStats.size() - 1).whiteNoiseLevel.getdBaLevels().length;
    double[] pearsonCoefficient = new double[dataSetCount];

    StringBuilder log = new StringBuilder();
    for(int freqId = 0; freqId < dataSetCount; freqId++) {
        double[] xValues = new double[freqLeqStats.size()];
        double[] yValues = new double[freqLeqStats.size()];
        int idStep = 0;
        for(LinearCalibrationResult result : freqLeqStats) {
            double dbLevel = result.measure[freqId].getLeqMean();
            double whiteNoise = result.whiteNoiseLevel.getdBaLevels()[freqId];
            xValues[idStep] = whiteNoise;
            yValues[idStep] = dbLevel;
            if(freqId == 0) {
                LOGGER.info("100 hZ white noise " + whiteNoise + " dB spl: " + dbLevel+ " dB");
            }
            idStep++;
        }
        pearsonCoefficient[freqId] = new PearsonsCorrelation().correlation(xValues, yValues);
        if(log.length() == 0) {
            log.append("[");
        } else {
            log.append(", ");
        }
        log.append(String.format(Locale.getDefault(), "%.2f %%",pearsonCoefficient[freqId] * 100 ));
    }
    log.append("]");
    LOGGER.info("Pearson's values : "+log.toString());
    return pearsonCoefficient;
}
 

@Test
public void testCorrelation2() {
  double[] x = new double[] {1, 2, 3, 4, 5, 6, 7, NaN, 9, 10};
  double[] y = new double[] {1, 2, 3, NaN, 5, 6, 7, 8, 9, 10};

  DoubleColumn xCol = DoubleColumn.create("x", x);
  DoubleColumn yCol = DoubleColumn.create("y", y);

  double resultP = xCol.pearsons(yCol);
  double resultK = xCol.kendalls(yCol);
  assertEquals(new PearsonsCorrelation().correlation(x, y), resultP, 0.0001);
  assertEquals(new KendallsCorrelation().correlation(x, y), resultK, 0.0001);
}
 

@Test
public void testCorrelation() {
  double[] x = new double[] {1, 2, 3, 4, 5, 6, 7, 8, 9, 10};
  double[] y = new double[] {1, 2, 3, 4, 5, 6, 7, 8, 9, 10};

  DoubleColumn xCol = DoubleColumn.create("x", x);
  DoubleColumn yCol = DoubleColumn.create("y", y);

  double resultP = xCol.pearsons(yCol);
  double resultS = xCol.spearmans(yCol);
  double resultK = xCol.kendalls(yCol);
  assertEquals(new PearsonsCorrelation().correlation(x, y), resultP, 0.0001);
  assertEquals(new SpearmansCorrelation().correlation(x, y), resultS, 0.0001);
  assertEquals(new KendallsCorrelation().correlation(x, y), resultK, 0.0001);
}
 

default double autoCorrelation(int lag) {
  int slice = this.size() - lag;
  if (slice <= 1) {
    return Double.NaN;
  }
  NumericColumn<?> x = (NumericColumn<?>) this.first(slice);
  NumericColumn<?> y = (NumericColumn<?>) this.last(slice);
  return new PearsonsCorrelation().correlation(x.asDoubleArray(), y.asDoubleArray());
}
 

@Test
public void testCorrelation2() {
  double[] x = new double[] {1, 2, 3, 4, 5, 6, 7, NaN, 9, 10};
  double[] y = new double[] {1, 2, 3, NaN, 5, 6, 7, 8, 9, 10};

  DoubleColumn xCol = DoubleColumn.create("x", x);
  DoubleColumn yCol = DoubleColumn.create("y", y);

  double resultP = xCol.pearsons(yCol);
  double resultK = xCol.kendalls(yCol);
  assertEquals(new PearsonsCorrelation().correlation(x, y), resultP, 0.0001);
  assertEquals(new KendallsCorrelation().correlation(x, y), resultK, 0.0001);
}
 

@Test
public void testCorrelation() {
  double[] x = new double[] {1, 2, 3, 4, 5, 6, 7, 8, 9, 10};
  double[] y = new double[] {1, 2, 3, 4, 5, 6, 7, 8, 9, 10};

  DoubleColumn xCol = DoubleColumn.create("x", x);
  DoubleColumn yCol = DoubleColumn.create("y", y);

  double resultP = xCol.pearsons(yCol);
  double resultS = xCol.spearmans(yCol);
  double resultK = xCol.kendalls(yCol);
  assertEquals(new PearsonsCorrelation().correlation(x, y), resultP, 0.0001);
  assertEquals(new SpearmansCorrelation().correlation(x, y), resultS, 0.0001);
  assertEquals(new KendallsCorrelation().correlation(x, y), resultK, 0.0001);
}
 

default double autoCorrelation(int lag) {
  int slice = this.size() - lag;
  if (slice <= 1) {
    return Double.NaN;
  }
  NumericColumn<?> x = (NumericColumn<?>) this.first(slice);
  NumericColumn<?> y = (NumericColumn<?>) this.last(slice);
  return new PearsonsCorrelation().correlation(x.asDoubleArray(), y.asDoubleArray());
}
 

@Test
public void test() throws IOException {
  double[] l1 = new double[] {3.4, 4.5, 6.7};
  double[] l2 = new double[] {1.2, 3.2, 3};
  
  values.clear();
  values.put("l1", l1);
  values.put("l2", l2);
  
  Assert.assertEquals(new PearsonsCorrelation().correlation(l1, l2), factory.constructEvaluator("corr(l1,l2)").evaluate(new Tuple(values)));
}
 

@Override
protected Object getExpectedValue(int start, int length)
{
    if (length <= 1) {
        return null;
    }
    PearsonsCorrelation corr = new PearsonsCorrelation();
    return (float) corr.correlation(constructDoublePrimitiveArray(start + 2, length), constructDoublePrimitiveArray(start, length));
}
 

private void testNonTrivialAggregation(double[] y, double[] x)
{
    PearsonsCorrelation corr = new PearsonsCorrelation();
    double expected = corr.correlation(x, y);
    checkArgument(Double.isFinite(expected) && expected != 0.0 && expected != 1.0, "Expected result is trivial");
    testAggregation(expected, createDoublesBlock(box(y)), createDoublesBlock(box(x)));
}
 

/**
 * Calculates a Pearson correlation coefficient.
 *
 * @param x X data
 * @param y Y data
 * @return Pearson correlation and p-value.
 */
public static double[] pearsonr(Array x, Array y) {
    x = x.copyIfView();
    y = y.copyIfView();

    if (ArrayMath.containsNaN(x) || ArrayMath.containsNaN(y)) {
        Array[] xy = ArrayMath.removeNaN(x, y);
        if (xy == null) {
            return new double[]{Double.NaN, Double.NaN};
        }
        
        x = xy[0];
        y = xy[1];
    }
    
    if (MAMath.isEqual(x, y)) {
        return new double[]{1, 0};
    }
    
    int m = (int)x.getSize();
    int n = 1;
    double[][] aa = new double[m][n * 2];
    for (int i = 0; i < m; i++) {
        for (int j = 0; j < n * 2; j++) {
            if (j < n) {
                aa[i][j] = x.getDouble(i * n + j);
            } else {
                aa[i][j] = y.getDouble(i * n + j - n);
            }
        }
    }
    RealMatrix matrix = new Array2DRowRealMatrix(aa, false);
    PearsonsCorrelation pc = new PearsonsCorrelation(matrix);
    double r = pc.getCorrelationMatrix().getEntry(0, 1);
    double pvalue = pc.getCorrelationPValues().getEntry(0, 1);
    return new double[]{r, pvalue};
}
 

@UserFunction(value = "regression.linear.correlation")
@Description("Calculate Pearson's correlation coefficient between first and second data lists")
public double correlation(@Name("first") List<Double> first, @Name("second") List<Double> second) {
    double[] firstArray = doubleListToArray(first);
    double[] secondArray = doubleListToArray(second);
    return new PearsonsCorrelation().correlation(firstArray, secondArray);
}
 

/** Returns the pearson's correlation between the receiver and the otherColumn */
default double pearsons(NumericColumn<?> otherColumn) {
  double[] x = asDoubleArray();
  double[] y = otherColumn.asDoubleArray();
  return new PearsonsCorrelation().correlation(x, y);
}
 

@Override
public double calc(double[][] data) {
  PearsonsCorrelation corr = new PearsonsCorrelation();
  return corr.correlation(col(data, 0), col(data, 1));
}
 

@Override
public double calc(double[][] data) {
  PearsonsCorrelation corr = new PearsonsCorrelation();
  return corr.correlation(col(data, 0), col(data, 1));
}
 

public void calculatePearson(double[] x, double[] y){
	PearsonsCorrelation pCorrelation = new PearsonsCorrelation();
	double cor = pCorrelation.correlation(x, y);//take out false too
	System.out.println(cor);
}
 

@Test
public void testPCorr() {
    MillerUpdatingRegression instance = new MillerUpdatingRegression(4, false);
    double[][] x = new double[airdata[0].length][];
    double[] y = new double[airdata[0].length];
    double[] cp = new double[10];
    double[] yxcorr = new double[4];
    double[] diag = new double[4];
    double sumysq = 0.0;
    int off = 0;
    for (int i = 0; i < airdata[0].length; i++) {
        x[i] = new double[4];
        x[i][0] = 1.0;
        x[i][1] = FastMath.log(airdata[3][i]);
        x[i][2] = FastMath.log(airdata[4][i]);
        x[i][3] = airdata[5][i];
        y[i] = FastMath.log(airdata[2][i]);
        off = 0;
        for (int j = 0; j < 4; j++) {
            double tmp = x[i][j];
            for (int k = 0; k <= j; k++, off++) {
                cp[off] += tmp * x[i][k];
            }
            yxcorr[j] += tmp * y[i];
        }
        sumysq += y[i] * y[i];
    }
    PearsonsCorrelation pearson = new PearsonsCorrelation(x);
    RealMatrix corr = pearson.getCorrelationMatrix();
    off = 0;
    for (int i = 0; i < 4; i++, off += (i + 1)) {
        diag[i] = FastMath.sqrt(cp[off]);
    }

    instance.addObservations(x, y);
    double[] pc = instance.getPartialCorrelations(0);
    int idx = 0;
    off = 0;
    int off2 = 6;
    for (int i = 0; i < 4; i++) {
        for (int j = 0; j < i; j++) {
            if (FastMath.abs(pc[idx] - cp[off] / (diag[i] * diag[j])) > 1.0e-8) {
                Assert.fail("Failed cross products... i = " + i + " j = " + j);
            }
            ++idx;
            ++off;
        }
        ++off;
        if (FastMath.abs(pc[i+off2] - yxcorr[ i] / (FastMath.sqrt(sumysq) * diag[i])) > 1.0e-8) {
            Assert.fail("Assert.failed cross product i = " + i + " y");
        }
    }
    double[] pc2 = instance.getPartialCorrelations(1);

    idx = 0;

    for (int i = 1; i < 4; i++) {
        for (int j = 1; j < i; j++) {
            if (FastMath.abs(pc2[idx] - corr.getEntry(j, i)) > 1.0e-8) {
                Assert.fail("Failed cross products... i = " + i + " j = " + j);
            }
            ++idx;
        }
    }
    double[] pc3 = instance.getPartialCorrelations(2);
    if (pc3 == null) {
        Assert.fail("Should not be null");
    }
    return;
}
 

@Test
public void testPCorr() {
    MillerUpdatingRegression instance = new MillerUpdatingRegression(4, false);
    double[][] x = new double[airdata[0].length][];
    double[] y = new double[airdata[0].length];
    double[] cp = new double[10];
    double[] yxcorr = new double[4];
    double[] diag = new double[4];
    double sumysq = 0.0;
    int off = 0;
    for (int i = 0; i < airdata[0].length; i++) {
        x[i] = new double[4];
        x[i][0] = 1.0;
        x[i][1] = Math.log(airdata[3][i]);
        x[i][2] = Math.log(airdata[4][i]);
        x[i][3] = airdata[5][i];
        y[i] = Math.log(airdata[2][i]);
        off = 0;
        for (int j = 0; j < 4; j++) {
            double tmp = x[i][j];
            for (int k = 0; k <= j; k++, off++) {
                cp[off] += tmp * x[i][k];
            }
            yxcorr[j] += tmp * y[i];
        }
        sumysq += y[i] * y[i];
    }
    PearsonsCorrelation pearson = new PearsonsCorrelation(x);
    RealMatrix corr = pearson.getCorrelationMatrix();
    off = 0;
    for (int i = 0; i < 4; i++, off += (i + 1)) {
        diag[i] = FastMath.sqrt(cp[off]);
    }

    instance.addObservations(x, y);
    double[] pc = instance.getPartialCorrelations(0);
    int idx = 0;
    off = 0;
    int off2 = 6;
    for (int i = 0; i < 4; i++) {
        for (int j = 0; j < i; j++) {
            if (Math.abs(pc[idx] - cp[off] / (diag[i] * diag[j])) > 1.0e-8) {
                Assert.fail("Failed cross products... i = " + i + " j = " + j);
            }
            ++idx;
            ++off;
        }
        ++off;
        if (Math.abs(pc[i+off2] - yxcorr[ i] / (FastMath.sqrt(sumysq) * diag[i])) > 1.0e-8) {
            Assert.fail("Assert.failed cross product i = " + i + " y");
        }
    }
    double[] pc2 = instance.getPartialCorrelations(1);

    idx = 0;

    for (int i = 1; i < 4; i++) {
        for (int j = 1; j < i; j++) {
            if (Math.abs(pc2[idx] - corr.getEntry(j, i)) > 1.0e-8) {
                Assert.fail("Failed cross products... i = " + i + " j = " + j);
            }
            ++idx;
        }
    }
    double[] pc3 = instance.getPartialCorrelations(2);
    if (pc3 == null) {
        Assert.fail("Should not be null");
    }
    return;
}
 

@Test
public void testPCorr() {
    MillerUpdatingRegression instance = new MillerUpdatingRegression(4, false);
    double[][] x = new double[airdata[0].length][];
    double[] y = new double[airdata[0].length];
    double[] cp = new double[10];
    double[] yxcorr = new double[4];
    double[] diag = new double[4];
    double sumysq = 0.0;
    int off = 0;
    for (int i = 0; i < airdata[0].length; i++) {
        x[i] = new double[4];
        x[i][0] = 1.0;
        x[i][1] = Math.log(airdata[3][i]);
        x[i][2] = Math.log(airdata[4][i]);
        x[i][3] = airdata[5][i];
        y[i] = Math.log(airdata[2][i]);
        off = 0;
        for (int j = 0; j < 4; j++) {
            double tmp = x[i][j];
            for (int k = 0; k <= j; k++, off++) {
                cp[off] += tmp * x[i][k];
            }
            yxcorr[j] += tmp * y[i];
        }
        sumysq += y[i] * y[i];
    }
    PearsonsCorrelation pearson = new PearsonsCorrelation(x);
    RealMatrix corr = pearson.getCorrelationMatrix();
    off = 0;
    for (int i = 0; i < 4; i++, off += (i + 1)) {
        diag[i] = FastMath.sqrt(cp[off]);
    }

    instance.addObservations(x, y);
    double[] pc = instance.getPartialCorrelations(0);
    int idx = 0;
    off = 0;
    int off2 = 6;
    for (int i = 0; i < 4; i++) {
        for (int j = 0; j < i; j++) {
            if (Math.abs(pc[idx] - cp[off] / (diag[i] * diag[j])) > 1.0e-8) {
                Assert.fail("Failed cross products... i = " + i + " j = " + j);
            }
            ++idx;
            ++off;
        }
        ++off;
        if (Math.abs(pc[i+off2] - yxcorr[ i] / (FastMath.sqrt(sumysq) * diag[i])) > 1.0e-8) {
            Assert.fail("Assert.failed cross product i = " + i + " y");
        }
    }
    double[] pc2 = instance.getPartialCorrelations(1);

    idx = 0;

    for (int i = 1; i < 4; i++) {
        for (int j = 1; j < i; j++) {
            if (Math.abs(pc2[idx] - corr.getEntry(j, i)) > 1.0e-8) {
                Assert.fail("Failed cross products... i = " + i + " j = " + j);
            }
            ++idx;
        }
    }
    double[] pc3 = instance.getPartialCorrelations(2);
    if (pc3 == null) {
        Assert.fail("Should not be null");
    }
    return;
}
 

@Test
public void testPCorr() {
    MillerUpdatingRegression instance = new MillerUpdatingRegression(4, false);
    double[][] x = new double[airdata[0].length][];
    double[] y = new double[airdata[0].length];
    double[] cp = new double[10];
    double[] yxcorr = new double[4];
    double[] diag = new double[4];
    double sumysq = 0.0;
    int off = 0;
    for (int i = 0; i < airdata[0].length; i++) {
        x[i] = new double[4];
        x[i][0] = 1.0;
        x[i][1] = Math.log(airdata[3][i]);
        x[i][2] = Math.log(airdata[4][i]);
        x[i][3] = airdata[5][i];
        y[i] = Math.log(airdata[2][i]);
        off = 0;
        for (int j = 0; j < 4; j++) {
            double tmp = x[i][j];
            for (int k = 0; k <= j; k++, off++) {
                cp[off] += tmp * x[i][k];
            }
            yxcorr[j] += tmp * y[i];
        }
        sumysq += y[i] * y[i];
    }
    PearsonsCorrelation pearson = new PearsonsCorrelation(x);
    RealMatrix corr = pearson.getCorrelationMatrix();
    off = 0;
    for (int i = 0; i < 4; i++, off += (i + 1)) {
        diag[i] = FastMath.sqrt(cp[off]);
    }

    instance.addObservations(x, y);
    double[] pc = instance.getPartialCorrelations(0);
    int idx = 0;
    off = 0;
    int off2 = 6;
    for (int i = 0; i < 4; i++) {
        for (int j = 0; j < i; j++) {
            if (Math.abs(pc[idx] - cp[off] / (diag[i] * diag[j])) > 1.0e-8) {
                Assert.fail("Failed cross products... i = " + i + " j = " + j);
            }
            ++idx;
            ++off;
        }
        ++off;
        if (Math.abs(pc[i+off2] - yxcorr[ i] / (FastMath.sqrt(sumysq) * diag[i])) > 1.0e-8) {
            Assert.fail("Assert.failed cross product i = " + i + " y");
        }
    }
    double[] pc2 = instance.getPartialCorrelations(1);

    idx = 0;

    for (int i = 1; i < 4; i++) {
        for (int j = 1; j < i; j++) {
            if (Math.abs(pc2[idx] - corr.getEntry(j, i)) > 1.0e-8) {
                Assert.fail("Failed cross products... i = " + i + " j = " + j);
            }
            ++idx;
        }
    }
    double[] pc3 = instance.getPartialCorrelations(2);
    if (pc3 == null) {
        Assert.fail("Should not be null");
    }
    return;
}
 

@Test
public void testPCorr() {
    MillerUpdatingRegression instance = new MillerUpdatingRegression(4, false);
    double[][] x = new double[airdata[0].length][];
    double[] y = new double[airdata[0].length];
    double[] cp = new double[10];
    double[] yxcorr = new double[4];
    double[] diag = new double[4];
    double sumysq = 0.0;
    int off = 0;
    for (int i = 0; i < airdata[0].length; i++) {
        x[i] = new double[4];
        x[i][0] = 1.0;
        x[i][1] = Math.log(airdata[3][i]);
        x[i][2] = Math.log(airdata[4][i]);
        x[i][3] = airdata[5][i];
        y[i] = Math.log(airdata[2][i]);
        off = 0;
        for (int j = 0; j < 4; j++) {
            double tmp = x[i][j];
            for (int k = 0; k <= j; k++, off++) {
                cp[off] += tmp * x[i][k];
            }
            yxcorr[j] += tmp * y[i];
        }
        sumysq += y[i] * y[i];
    }
    PearsonsCorrelation pearson = new PearsonsCorrelation(x);
    RealMatrix corr = pearson.getCorrelationMatrix();
    off = 0;
    for (int i = 0; i < 4; i++, off += (i + 1)) {
        diag[i] = FastMath.sqrt(cp[off]);
    }

    instance.addObservations(x, y);
    double[] pc = instance.getPartialCorrelations(0);
    int idx = 0;
    off = 0;
    int off2 = 6;
    for (int i = 0; i < 4; i++) {
        for (int j = 0; j < i; j++) {
            if (Math.abs(pc[idx] - cp[off] / (diag[i] * diag[j])) > 1.0e-8) {
                Assert.fail("Failed cross products... i = " + i + " j = " + j);
            }
            ++idx;
            ++off;
        }
        ++off;
        if (Math.abs(pc[i+off2] - yxcorr[ i] / (FastMath.sqrt(sumysq) * diag[i])) > 1.0e-8) {
            Assert.fail("Assert.failed cross product i = " + i + " y");
        }
    }
    double[] pc2 = instance.getPartialCorrelations(1);

    idx = 0;

    for (int i = 1; i < 4; i++) {
        for (int j = 1; j < i; j++) {
            if (Math.abs(pc2[idx] - corr.getEntry(j, i)) > 1.0e-8) {
                Assert.fail("Failed cross products... i = " + i + " j = " + j);
            }
            ++idx;
        }
    }
    double[] pc3 = instance.getPartialCorrelations(2);
    if (pc3 == null) {
        Assert.fail("Should not be null");
    }
    return;
}
 

@Test
public void testPCorr() {
    MillerUpdatingRegression instance = new MillerUpdatingRegression(4, false);
    double[][] x = new double[airdata[0].length][];
    double[] y = new double[airdata[0].length];
    double[] cp = new double[10];
    double[] yxcorr = new double[4];
    double[] diag = new double[4];
    double sumysq = 0.0;
    int off = 0;
    for (int i = 0; i < airdata[0].length; i++) {
        x[i] = new double[4];
        x[i][0] = 1.0;
        x[i][1] = Math.log(airdata[3][i]);
        x[i][2] = Math.log(airdata[4][i]);
        x[i][3] = airdata[5][i];
        y[i] = Math.log(airdata[2][i]);
        off = 0;
        for (int j = 0; j < 4; j++) {
            double tmp = x[i][j];
            for (int k = 0; k <= j; k++, off++) {
                cp[off] += tmp * x[i][k];
            }
            yxcorr[j] += tmp * y[i];
        }
        sumysq += y[i] * y[i];
    }
    PearsonsCorrelation pearson = new PearsonsCorrelation(x);
    RealMatrix corr = pearson.getCorrelationMatrix();
    off = 0;
    for (int i = 0; i < 4; i++, off += (i + 1)) {
        diag[i] = FastMath.sqrt(cp[off]);
    }

    instance.addObservations(x, y);
    double[] pc = instance.getPartialCorrelations(0);
    int idx = 0;
    off = 0;
    int off2 = 6;
    for (int i = 0; i < 4; i++) {
        for (int j = 0; j < i; j++) {
            if (Math.abs(pc[idx] - cp[off] / (diag[i] * diag[j])) > 1.0e-8) {
                Assert.fail("Failed cross products... i = " + i + " j = " + j);
            }
            ++idx;
            ++off;
        }
        ++off;
        if (Math.abs(pc[i+off2] - yxcorr[ i] / (FastMath.sqrt(sumysq) * diag[i])) > 1.0e-8) {
            Assert.fail("Assert.failed cross product i = " + i + " y");
        }
    }
    double[] pc2 = instance.getPartialCorrelations(1);

    idx = 0;

    for (int i = 1; i < 4; i++) {
        for (int j = 1; j < i; j++) {
            if (Math.abs(pc2[idx] - corr.getEntry(j, i)) > 1.0e-8) {
                Assert.fail("Failed cross products... i = " + i + " j = " + j);
            }
            ++idx;
        }
    }
    double[] pc3 = instance.getPartialCorrelations(2);
    if (pc3 == null) {
        Assert.fail("Should not be null");
    }
    return;
}
 

@Test
public void testPCorr() {
    MillerUpdatingRegression instance = new MillerUpdatingRegression(4, false);
    double[][] x = new double[airdata[0].length][];
    double[] y = new double[airdata[0].length];
    double[] cp = new double[10];
    double[] yxcorr = new double[4];
    double[] diag = new double[4];
    double sumysq = 0.0;
    int off = 0;
    for (int i = 0; i < airdata[0].length; i++) {
        x[i] = new double[4];
        x[i][0] = 1.0;
        x[i][1] = Math.log(airdata[3][i]);
        x[i][2] = Math.log(airdata[4][i]);
        x[i][3] = airdata[5][i];
        y[i] = Math.log(airdata[2][i]);
        off = 0;
        for (int j = 0; j < 4; j++) {
            double tmp = x[i][j];
            for (int k = 0; k <= j; k++, off++) {
                cp[off] += tmp * x[i][k];
            }
            yxcorr[j] += tmp * y[i];
        }
        sumysq += y[i] * y[i];
    }
    PearsonsCorrelation pearson = new PearsonsCorrelation(x);
    RealMatrix corr = pearson.getCorrelationMatrix();
    off = 0;
    for (int i = 0; i < 4; i++, off += (i + 1)) {
        diag[i] = FastMath.sqrt(cp[off]);
    }

    instance.addObservations(x, y);
    double[] pc = instance.getPartialCorrelations(0);
    int idx = 0;
    off = 0;
    int off2 = 6;
    for (int i = 0; i < 4; i++) {
        for (int j = 0; j < i; j++) {
            if (Math.abs(pc[idx] - cp[off] / (diag[i] * diag[j])) > 1.0e-8) {
                Assert.fail("Failed cross products... i = " + i + " j = " + j);
            }
            ++idx;
            ++off;
        }
        ++off;
        if (Math.abs(pc[i+off2] - yxcorr[ i] / (FastMath.sqrt(sumysq) * diag[i])) > 1.0e-8) {
            Assert.fail("Assert.failed cross product i = " + i + " y");
        }
    }
    double[] pc2 = instance.getPartialCorrelations(1);

    idx = 0;

    for (int i = 1; i < 4; i++) {
        for (int j = 1; j < i; j++) {
            if (Math.abs(pc2[idx] - corr.getEntry(j, i)) > 1.0e-8) {
                Assert.fail("Failed cross products... i = " + i + " j = " + j);
            }
            ++idx;
        }
    }
    double[] pc3 = instance.getPartialCorrelations(2);
    if (pc3 == null) {
        Assert.fail("Should not be null");
    }
    return;
}
 

@Test
public void testPCorr() {
    MillerUpdatingRegression instance = new MillerUpdatingRegression(4, false);
    double[][] x = new double[airdata[0].length][];
    double[] y = new double[airdata[0].length];
    double[] cp = new double[10];
    double[] yxcorr = new double[4];
    double[] diag = new double[4];
    double sumysq = 0.0;
    int off = 0;
    for (int i = 0; i < airdata[0].length; i++) {
        x[i] = new double[4];
        x[i][0] = 1.0;
        x[i][1] = FastMath.log(airdata[3][i]);
        x[i][2] = FastMath.log(airdata[4][i]);
        x[i][3] = airdata[5][i];
        y[i] = FastMath.log(airdata[2][i]);
        off = 0;
        for (int j = 0; j < 4; j++) {
            double tmp = x[i][j];
            for (int k = 0; k <= j; k++, off++) {
                cp[off] += tmp * x[i][k];
            }
            yxcorr[j] += tmp * y[i];
        }
        sumysq += y[i] * y[i];
    }
    PearsonsCorrelation pearson = new PearsonsCorrelation(x);
    RealMatrix corr = pearson.getCorrelationMatrix();
    off = 0;
    for (int i = 0; i < 4; i++, off += (i + 1)) {
        diag[i] = FastMath.sqrt(cp[off]);
    }

    instance.addObservations(x, y);
    double[] pc = instance.getPartialCorrelations(0);
    int idx = 0;
    off = 0;
    int off2 = 6;
    for (int i = 0; i < 4; i++) {
        for (int j = 0; j < i; j++) {
            if (FastMath.abs(pc[idx] - cp[off] / (diag[i] * diag[j])) > 1.0e-8) {
                Assert.fail("Failed cross products... i = " + i + " j = " + j);
            }
            ++idx;
            ++off;
        }
        ++off;
        if (FastMath.abs(pc[i+off2] - yxcorr[ i] / (FastMath.sqrt(sumysq) * diag[i])) > 1.0e-8) {
            Assert.fail("Assert.failed cross product i = " + i + " y");
        }
    }
    double[] pc2 = instance.getPartialCorrelations(1);

    idx = 0;

    for (int i = 1; i < 4; i++) {
        for (int j = 1; j < i; j++) {
            if (FastMath.abs(pc2[idx] - corr.getEntry(j, i)) > 1.0e-8) {
                Assert.fail("Failed cross products... i = " + i + " j = " + j);
            }
            ++idx;
        }
    }
    double[] pc3 = instance.getPartialCorrelations(2);
    if (pc3 == null) {
        Assert.fail("Should not be null");
    }
    return;
}
 

/** Returns the pearson's correlation between the receiver and the otherColumn */
default double pearsons(NumericColumn<?> otherColumn) {
  double[] x = asDoubleArray();
  double[] y = otherColumn.asDoubleArray();
  return new PearsonsCorrelation().correlation(x, y);
}