org.apache.commons.math.stat.StatUtils Java Examples

public void testMinMax() {
    da.addElement(2.0);
    da.addElement(22.0);
    da.addElement(-2.0);
    da.addElement(21.0);
    da.addElement(22.0);
    da.addElement(42.0);
    da.addElement(62.0);
    da.addElement(22.0);
    da.addElement(122.0);
    da.addElement(1212.0);

    assertEquals("Min should be -2.0", -2.0, StatUtils.min(da.getElements()), Double.MIN_VALUE);
    assertEquals(
        "Max should be 1212.0",
        1212.0,
        StatUtils.max(da.getElements()),
        Double.MIN_VALUE);
}
 

@Test
public void testSample() {
    final double[] values = { -2.0d, 2.0d, 4.0d, -2.0d, 22.0d, 11.0d, 3.0d, 14.0d, 5.0d };
    final int length = values.length;
    final double mean = StatUtils.mean(values); // 6.333...
    final SemiVariance sv = new SemiVariance();  // Default bias correction is true
    final double downsideSemiVariance = sv.evaluate(values); // Downside is the default
    Assert.assertEquals(TestUtils.sumSquareDev(new double[] {-2d, 2d, 4d, -2d, 3d, 5d}, mean) / (length - 1),
            downsideSemiVariance, 1E-14);

    sv.setVarianceDirection(SemiVariance.UPSIDE_VARIANCE);
    final double upsideSemiVariance = sv.evaluate(values);
    Assert.assertEquals(TestUtils.sumSquareDev(new double[] {22d, 11d, 14d}, mean) / (length - 1),
            upsideSemiVariance, 1E-14);

    // Verify that upper + lower semivariance against the mean sum to variance
    Assert.assertEquals(StatUtils.variance(values), downsideSemiVariance + upsideSemiVariance, 10e-12);
}
 

private void computePercentiles() {

        // Statistics, other attributes
        DoubleArrayList flattenedDataList = new DoubleArrayList(matrix.getColumnDimension() * matrix.getRowDimension());
        double min = 1;
        double max = -1;
        for (int i = 0; i < matrix.getRowDimension(); i++) {
            for (int j = 0; j < matrix.getColumnDimension(); j++) {
                double value = matrix.getEntry(i, j);
                if (!Double.isNaN(value) && value != 1) {
                    min = value < min ? value : min;
                    max = value > max ? value : max;
                    flattenedDataList.add(value);
                }
            }
        }

        // Stats
        double[] flattenedData = flattenedDataList.toArray();
        lowerValue = (float) StatUtils.percentile(flattenedData, 5);
        upperValue = (float) StatUtils.percentile(flattenedData, 95);
        System.out.println(lowerValue + "  " + upperValue);

    }
 

public void testMinMax() {
    da.addElement(2.0);
    da.addElement(22.0);
    da.addElement(-2.0);
    da.addElement(21.0);
    da.addElement(22.0);
    da.addElement(42.0);
    da.addElement(62.0);
    da.addElement(22.0);
    da.addElement(122.0);
    da.addElement(1212.0);

    assertEquals("Min should be -2.0", -2.0, StatUtils.min(da.getElements()), Double.MIN_VALUE);
    assertEquals(
        "Max should be 1212.0",
        1212.0,
        StatUtils.max(da.getElements()),
        Double.MIN_VALUE);
}
 

public void testSample() {
    final double[] values = { -2.0d, 2.0d, 4.0d, -2.0d, 22.0d, 11.0d, 3.0d, 14.0d, 5.0d };
    final int length = values.length;
    final double mean = StatUtils.mean(values); // 6.333...
    final SemiVariance sv = new SemiVariance();  // Default bias correction is true
    final double downsideSemiVariance = sv.evaluate(values); // Downside is the default
    assertEquals(TestUtils.sumSquareDev(new double[] {-2d, 2d, 4d, -2d, 3d, 5d}, mean) / (length - 1),
            downsideSemiVariance, 1E-14);

    sv.setVarianceDirection(SemiVariance.UPSIDE_VARIANCE);
    final double upsideSemiVariance = sv.evaluate(values);
    assertEquals(TestUtils.sumSquareDev(new double[] {22d, 11d, 14d}, mean) / (length - 1),
            upsideSemiVariance, 1E-14);

    // Verify that upper + lower semivariance against the mean sum to variance
    assertEquals(StatUtils.variance(values), downsideSemiVariance + upsideSemiVariance, 10e-12);
}
 

public void testSample() {
    final double[] values = { -2.0d, 2.0d, 4.0d, -2.0d, 22.0d, 11.0d, 3.0d, 14.0d, 5.0d };
    final int length = values.length;
    final double mean = StatUtils.mean(values); // 6.333...
    final SemiVariance sv = new SemiVariance();  // Default bias correction is true
    final double downsideSemiVariance = sv.evaluate(values); // Downside is the default
    assertEquals(TestUtils.sumSquareDev(new double[] {-2d, 2d, 4d, -2d, 3d, 5d}, mean) / (length - 1),
            downsideSemiVariance, 1E-14);

    sv.setVarianceDirection(SemiVariance.UPSIDE_VARIANCE);
    final double upsideSemiVariance = sv.evaluate(values);
    assertEquals(TestUtils.sumSquareDev(new double[] {22d, 11d, 14d}, mean) / (length - 1),
            upsideSemiVariance, 1E-14);

    // Verify that upper + lower semivariance against the mean sum to variance
    assertEquals(StatUtils.variance(values), downsideSemiVariance + upsideSemiVariance, 10e-12);
}
 

public void testMinMax() {
    da.addElement(2.0);
    da.addElement(22.0);
    da.addElement(-2.0);
    da.addElement(21.0);
    da.addElement(22.0);
    da.addElement(42.0);
    da.addElement(62.0);
    da.addElement(22.0);
    da.addElement(122.0);
    da.addElement(1212.0);

    assertEquals("Min should be -2.0", -2.0, StatUtils.min(da.getElements()), Double.MIN_VALUE);
    assertEquals(
        "Max should be 1212.0",
        1212.0,
        StatUtils.max(da.getElements()),
        Double.MIN_VALUE);
}
 

public void testMinMax() {
    da.addElement(2.0);
    da.addElement(22.0);
    da.addElement(-2.0);
    da.addElement(21.0);
    da.addElement(22.0);
    da.addElement(42.0);
    da.addElement(62.0);
    da.addElement(22.0);
    da.addElement(122.0);
    da.addElement(1212.0);

    assertEquals("Min should be -2.0", -2.0, StatUtils.min(da.getElements()), Double.MIN_VALUE);
    assertEquals(
        "Max should be 1212.0",
        1212.0,
        StatUtils.max(da.getElements()),
        Double.MIN_VALUE);
}
 

public void testSample() {
    final double[] values = { -2.0d, 2.0d, 4.0d, -2.0d, 22.0d, 11.0d, 3.0d, 14.0d, 5.0d };
    final int length = values.length;
    final double mean = StatUtils.mean(values); // 6.333...
    final SemiVariance sv = new SemiVariance();  // Default bias correction is true
    final double downsideSemiVariance = sv.evaluate(values); // Downside is the default
    assertEquals(TestUtils.sumSquareDev(new double[] {-2d, 2d, 4d, -2d, 3d, 5d}, mean) / (length - 1),
            downsideSemiVariance, 1E-14);

    sv.setVarianceDirection(SemiVariance.UPSIDE_VARIANCE);
    final double upsideSemiVariance = sv.evaluate(values);
    assertEquals(TestUtils.sumSquareDev(new double[] {22d, 11d, 14d}, mean) / (length - 1),
            upsideSemiVariance, 1E-14);

    // Verify that upper + lower semivariance against the mean sum to variance
    assertEquals(StatUtils.variance(values), downsideSemiVariance + upsideSemiVariance, 10e-12);
}
 

@Test
public void testSample() {
    final double[] values = { -2.0d, 2.0d, 4.0d, -2.0d, 22.0d, 11.0d, 3.0d, 14.0d, 5.0d };
    final int length = values.length;
    final double mean = StatUtils.mean(values); // 6.333...
    final SemiVariance sv = new SemiVariance();  // Default bias correction is true
    final double downsideSemiVariance = sv.evaluate(values); // Downside is the default
    Assert.assertEquals(TestUtils.sumSquareDev(new double[] {-2d, 2d, 4d, -2d, 3d, 5d}, mean) / (length - 1),
            downsideSemiVariance, 1E-14);

    sv.setVarianceDirection(SemiVariance.UPSIDE_VARIANCE);
    final double upsideSemiVariance = sv.evaluate(values);
    Assert.assertEquals(TestUtils.sumSquareDev(new double[] {22d, 11d, 14d}, mean) / (length - 1),
            upsideSemiVariance, 1E-14);

    // Verify that upper + lower semivariance against the mean sum to variance
    Assert.assertEquals(StatUtils.variance(values), downsideSemiVariance + upsideSemiVariance, 10e-12);
}
 

public void testMinMax() {
    da.addElement(2.0);
    da.addElement(22.0);
    da.addElement(-2.0);
    da.addElement(21.0);
    da.addElement(22.0);
    da.addElement(42.0);
    da.addElement(62.0);
    da.addElement(22.0);
    da.addElement(122.0);
    da.addElement(1212.0);

    assertEquals("Min should be -2.0", -2.0, StatUtils.min(da.getElements()), Double.MIN_VALUE);
    assertEquals(
        "Max should be 1212.0",
        1212.0,
        StatUtils.max(da.getElements()),
        Double.MIN_VALUE);
}
 

@Test
public void testMinMax() {
    da.addElement(2.0);
    da.addElement(22.0);
    da.addElement(-2.0);
    da.addElement(21.0);
    da.addElement(22.0);
    da.addElement(42.0);
    da.addElement(62.0);
    da.addElement(22.0);
    da.addElement(122.0);
    da.addElement(1212.0);

    Assert.assertEquals("Min should be -2.0", -2.0, StatUtils.min(da.getElements()), Double.MIN_VALUE);
    Assert.assertEquals(
        "Max should be 1212.0",
        1212.0,
        StatUtils.max(da.getElements()),
        Double.MIN_VALUE);
}
 

/** test stats */
public void testSerialization() {
    
    double[] values = {35d, 23d, 42d};
    
    DescriptiveStatistics u = new BeanListUnivariateImpl( patientList, "age" ); 
    assertEquals("total count",3,u.getN(),tolerance);
    assertEquals("mean", StatUtils.mean(values), u.getMean(), tolerance);
    assertEquals("min", StatUtils.min(values), u.getMin(), tolerance);
    assertEquals("max", StatUtils.max(values), u.getMax(), tolerance);
    assertEquals("var", StatUtils.variance(values), u.getVariance(), tolerance);   
    
    
    DescriptiveStatistics u2 = (DescriptiveStatistics)TestUtils.serializeAndRecover(u); 
    assertEquals("total count",3,u2.getN(),tolerance);
    assertEquals("mean", StatUtils.mean(values), u2.getMean(), tolerance);
    assertEquals("min", StatUtils.min(values), u2.getMin(), tolerance);
    assertEquals("max", StatUtils.max(values), u2.getMax(), tolerance);
    assertEquals("var", StatUtils.variance(values), u2.getVariance(), tolerance);   

    u.clear();
    assertEquals("total count",0,u.getN(),tolerance);    
    
    u2.clear();
    assertEquals("total count",0,u2.getN(),tolerance);
        
}
 

/**
 * Check that the lower + upper semivariance against the mean sum to the
 * variance.
 */
public void testVarianceDecompMeanCutoff() {
    double[] values = { -2.0d, 2.0d, 4.0d, -2.0d, 22.0d, 11.0d, 3.0d, 14.0d, 5.0d };
    double variance = StatUtils.variance(values);
    SemiVariance sv = new SemiVariance(true); // Bias corrected
    sv.setVarianceDirection(SemiVariance.DOWNSIDE_VARIANCE);
    final double lower = sv.evaluate(values);
    sv.setVarianceDirection(SemiVariance.UPSIDE_VARIANCE);
    final double upper = sv.evaluate(values);
    assertEquals(variance, lower + upper, 10e-12);
}
 

public void testMeanAndStandardDeviation() {
    RandomGenerator rg = new JDKRandomGenerator();
    rg.setSeed(17399225432l);
    UniformRandomGenerator generator = new UniformRandomGenerator(rg);
    double[] sample = new double[10000];
    for (int i = 0; i < sample.length; ++i) {
        sample[i] = generator.nextNormalizedDouble();
    }
    assertEquals(0.0, StatUtils.mean(sample), 0.07);
    assertEquals(1.0, StatUtils.variance(sample), 0.02);
}
 

public void testMeanAndStandardDeviation() {
    RandomGenerator rg = new JDKRandomGenerator();
    rg.setSeed(17399225432l);
    UniformRandomGenerator generator = new UniformRandomGenerator(rg);
    double[] sample = new double[10000];
    for (int i = 0; i < sample.length; ++i) {
        sample[i] = generator.nextNormalizedDouble();
    }
    assertEquals(0.0, StatUtils.mean(sample), 0.07);
    assertEquals(1.0, StatUtils.variance(sample), 0.02);
}
 

/** test stats */
public void testSerialization() {
    
    double[] values = {35d, 23d, 42d};
    
    DescriptiveStatistics u = new BeanListUnivariateImpl( patientList, "age" ); 
    assertEquals("total count",3,u.getN(),tolerance);
    assertEquals("mean", StatUtils.mean(values), u.getMean(), tolerance);
    assertEquals("min", StatUtils.min(values), u.getMin(), tolerance);
    assertEquals("max", StatUtils.max(values), u.getMax(), tolerance);
    assertEquals("var", StatUtils.variance(values), u.getVariance(), tolerance);   
    
    
    DescriptiveStatistics u2 = (DescriptiveStatistics)TestUtils.serializeAndRecover(u); 
    assertEquals("total count",3,u2.getN(),tolerance);
    assertEquals("mean", StatUtils.mean(values), u2.getMean(), tolerance);
    assertEquals("min", StatUtils.min(values), u2.getMin(), tolerance);
    assertEquals("max", StatUtils.max(values), u2.getMax(), tolerance);
    assertEquals("var", StatUtils.variance(values), u2.getVariance(), tolerance);   

    u.clear();
    assertEquals("total count",0,u.getN(),tolerance);    
    
    u2.clear();
    assertEquals("total count",0,u2.getN(),tolerance);
        
}
 

/**
 * Check that the lower + upper semivariance against the mean sum to the
 * variance.
 */
public void testVarianceDecompMeanCutoff() {
    double[] values = { -2.0d, 2.0d, 4.0d, -2.0d, 22.0d, 11.0d, 3.0d, 14.0d, 5.0d };
    double variance = StatUtils.variance(values);
    SemiVariance sv = new SemiVariance(true); // Bias corrected
    sv.setVarianceDirection(SemiVariance.DOWNSIDE_VARIANCE);
    final double lower = sv.evaluate(values);
    sv.setVarianceDirection(SemiVariance.UPSIDE_VARIANCE);
    final double upper = sv.evaluate(values);
    assertEquals(variance, lower + upper, 10e-12);
}
 

/** test stats */
public void testStats() {	
    DescriptiveStatistics u = new BeanListUnivariateImpl( patientList, "age" ); 
    double[] values = {35d, 23d, 42d};
    assertEquals("total count",3,u.getN(),tolerance);
    assertEquals("mean", StatUtils.mean(values), u.getMean(), tolerance);
    assertEquals("min", StatUtils.min(values), u.getMin(), tolerance);
    assertEquals("max", StatUtils.max(values), u.getMax(), tolerance);
    assertEquals("var", StatUtils.variance(values), u.getVariance(), tolerance);       
    u.clear();
    assertEquals("total count",0,u.getN(),tolerance);    
}
 

public void testMeanAndStandardDeviation() {
    RandomGenerator rg = new JDKRandomGenerator();
    rg.setSeed(17399225432l);
    UniformRandomGenerator generator = new UniformRandomGenerator(rg);
    double[] sample = new double[10000];
    for (int i = 0; i < sample.length; ++i) {
        sample[i] = generator.nextNormalizedDouble();
    }
    assertEquals(0.0, StatUtils.mean(sample), 0.07);
    assertEquals(1.0, StatUtils.variance(sample), 0.02);
}
 

public void testMeanAndStandardDeviation() {
    RandomGenerator rg = new JDKRandomGenerator();
    rg.setSeed(17399225432l);
    GaussianRandomGenerator generator = new GaussianRandomGenerator(rg);
    double[] sample = new double[10000];
    for (int i = 0; i < sample.length; ++i) {
        sample[i] = generator.nextNormalizedDouble();
    }
    assertEquals(0.0, StatUtils.mean(sample), 0.012);
    assertEquals(1.0, StatUtils.variance(sample), 0.01);
}
 

/**
 * Check that the lower + upper semivariance against the mean sum to the
 * variance.
 */
public void testVarianceDecompMeanCutoff() {
    double[] values = { -2.0d, 2.0d, 4.0d, -2.0d, 22.0d, 11.0d, 3.0d, 14.0d, 5.0d };
    double variance = StatUtils.variance(values);
    SemiVariance sv = new SemiVariance(true); // Bias corrected
    sv.setVarianceDirection(SemiVariance.DOWNSIDE_VARIANCE);
    final double lower = sv.evaluate(values);
    sv.setVarianceDirection(SemiVariance.UPSIDE_VARIANCE);
    final double upper = sv.evaluate(values);
    assertEquals(variance, lower + upper, 10e-12);
}
 

public static double getFeatureMatchDistribution(GraphDatabaseService db, Long patternId)
{
    Transaction tx = db.beginTx();
    Node startNode = db.getNodeById(patternId);

    // Feature match distribution
    List<Double> matches = IteratorUtil.asCollection(db.traversalDescription()
            .depthFirst()
            .relationships(withName("HAS_CLASS"), Direction.OUTGOING)
            .evaluator(Evaluators.fromDepth(1))
            .evaluator(Evaluators.toDepth(1))
            .traverse(startNode)
            .relationships())
            .stream()
            .map(p -> ((Integer)p.getProperty("matches")).doubleValue())
            .collect(Collectors.toList());

    tx.success();
    tx.close();

    double variance = 1.0;

    if(matches.size() > 1) {
        Double[] matchArr = matches.toArray(new Double[matches.size()]);
        // Get the standard deviation
        DescriptiveStatistics ds = new DescriptiveStatistics();
        matches.forEach(m -> ds.addValue(m.doubleValue() / StatUtils.sum(ArrayUtils.toPrimitive(matchArr))));
        variance = ds.getStandardDeviation();
    }

    return variance;
}
 

/**
 *
 * @param list
 * @return
 */
public double getVariance(List<SynsetOut> list) {
    double[] scores = new double[list.size()];
    int l = 0;
    for (SynsetOut out : list) {
        scores[l] = out.getScore();
        l++;
    }
    return StatUtils.variance(scores);
}
 

/**
 * Verifies that calculateYVariance and calculateResidualVariance return consistent
 * values with direct variance computation from Y, residuals, respectively.
 */
protected void checkVarianceConsistency(OLSMultipleLinearRegression model) throws Exception {
    // Check Y variance consistency
    TestUtils.assertEquals(StatUtils.variance(model.Y.getData()), model.calculateYVariance(), 0);
    
    // Check residual variance consistency
    double[] residuals = model.calculateResiduals().getData();
    RealMatrix X = model.X;
    TestUtils.assertEquals(
            StatUtils.variance(model.calculateResiduals().getData()) * (residuals.length - 1),
            model.calculateErrorVariance() * (X.getRowDimension() - X.getColumnDimension()), 1E-20);
    
}
 

/**
 * Check that the lower + upper semivariance against the mean sum to the
 * variance.
 */
public void testVarianceDecompMeanCutoff() {
    double[] values = { -2.0d, 2.0d, 4.0d, -2.0d, 22.0d, 11.0d, 3.0d, 14.0d, 5.0d };
    double variance = StatUtils.variance(values);
    SemiVariance sv = new SemiVariance(true); // Bias corrected
    sv.setVarianceDirection(SemiVariance.DOWNSIDE_VARIANCE);
    final double lower = sv.evaluate(values);
    sv.setVarianceDirection(SemiVariance.UPSIDE_VARIANCE);
    final double upper = sv.evaluate(values);
    assertEquals(variance, lower + upper, 10e-12);
}
 

public void testAddElementRolling() {
    ra.addElement(0.5);
    ra.addElement(1.0);
    ra.addElement(1.0);
    ra.addElement(1.0);
    ra.addElement(1.0);
    ra.addElement(1.0);
    ra.addElementRolling(2.0);

    assertEquals(
        "There should be 6 elements in the eda",
        6,
        ra.getNumElements());
    assertEquals(
        "The max element should be 2.0",
        2.0,
        StatUtils.max(ra.getElements()),
        Double.MIN_VALUE);
    assertEquals(
        "The min element should be 1.0",
        1.0,
        StatUtils.min(ra.getElements()),
        Double.MIN_VALUE);

    for (int i = 0; i < 1024; i++) {
        ra.addElementRolling(i);
    }

    assertEquals(
        "We just inserted 1024 rolling elements, num elements should still be 6",
        6,
        ra.getNumElements());
}
 

public void testMeanAndStandardDeviation() {
    RandomGenerator rg = new JDKRandomGenerator();
    rg.setSeed(17399225432l);
    UniformRandomGenerator generator = new UniformRandomGenerator(rg);
    double[] sample = new double[10000];
    for (int i = 0; i < sample.length; ++i) {
        sample[i] = generator.nextNormalizedDouble();
    }
    assertEquals(0.0, StatUtils.mean(sample), 0.07);
    assertEquals(1.0, StatUtils.variance(sample), 0.02);
}
 

public void testMeanAndStandardDeviation() {
    RandomGenerator rg = new JDKRandomGenerator();
    rg.setSeed(17399225432l);
    UniformRandomGenerator generator = new UniformRandomGenerator(rg);
    double[] sample = new double[10000];
    for (int i = 0; i < sample.length; ++i) {
        sample[i] = generator.nextNormalizedDouble();
    }
    assertEquals(0.0, StatUtils.mean(sample), 0.07);
    assertEquals(1.0, StatUtils.variance(sample), 0.02);
}
 

public void testMeanAndStandardDeviation() {
    RandomGenerator rg = new JDKRandomGenerator();
    rg.setSeed(17399225432l);
    GaussianRandomGenerator generator = new GaussianRandomGenerator(rg);
    double[] sample = new double[10000];
    for (int i = 0; i < sample.length; ++i) {
        sample[i] = generator.nextNormalizedDouble();
    }
    assertEquals(0.0, StatUtils.mean(sample), 0.012);
    assertEquals(1.0, StatUtils.variance(sample), 0.01);
}