org.apache.commons.math.stat.descriptive.SummaryStatistics Java Examples

/**
 * Test SummaryStatistics - implementations that do not store the data
 * and use single pass algorithms to compute statistics
*/
@Test
public void testSummaryStatistics() throws Exception {
    SummaryStatistics u = new SummaryStatistics();
    loadStats("data/PiDigits.txt", u);
    Assert.assertEquals("PiDigits: std", std, u.getStandardDeviation(), 1E-13);
    Assert.assertEquals("PiDigits: mean", mean, u.getMean(), 1E-13);

    loadStats("data/Mavro.txt", u);
    Assert.assertEquals("Mavro: std", std, u.getStandardDeviation(), 1E-14);
    Assert.assertEquals("Mavro: mean", mean, u.getMean(), 1E-14);

    loadStats("data/Michelso.txt", u);
    Assert.assertEquals("Michelso: std", std, u.getStandardDeviation(), 1E-13);
    Assert.assertEquals("Michelso: mean", mean, u.getMean(), 1E-13);

    loadStats("data/NumAcc1.txt", u);
    Assert.assertEquals("NumAcc1: std", std, u.getStandardDeviation(), 1E-14);
    Assert.assertEquals("NumAcc1: mean", mean, u.getMean(), 1E-14);

    loadStats("data/NumAcc2.txt", u);
    Assert.assertEquals("NumAcc2: std", std, u.getStandardDeviation(), 1E-14);
    Assert.assertEquals("NumAcc2: mean", mean, u.getMean(), 1E-14);
}
 

/**
 * Generates a random value from this distribution.
 * 
 * @return the random value.
 * @throws IllegalStateException if the distribution has not been loaded
 */
public double getNextValue() throws IllegalStateException {

    if (!loaded) {
        throw MathRuntimeException.createIllegalStateException("distribution not loaded");
    }

    // Start with a uniformly distributed random number in (0,1)
    double x = Math.random();

    // Use this to select the bin and generate a Gaussian within the bin
    for (int i = 0; i < binCount; i++) {
       if (x <= upperBounds[i]) {
           SummaryStatistics stats = binStats.get(i);
           if (stats.getN() > 0) {
               if (stats.getStandardDeviation() > 0) {  // more than one obs
                    return randomData.nextGaussian
                        (stats.getMean(),stats.getStandardDeviation());
               } else {
                   return stats.getMean(); // only one obs in bin
               }
           }
       }
    }
    throw new MathRuntimeException("no bin selected");
}
 

/** test failure modes and distribution of nextGaussian() */
public void testNextGaussian() {
    try {
        randomData.nextGaussian(0, 0);
        fail("zero sigma -- MathIllegalArgumentException expected");
    } catch (MathIllegalArgumentException ex) {
        // ignored
    }
    SummaryStatistics u = new SummaryStatistics();
    for (int i = 0; i < largeSampleSize; i++) {
        u.addValue(randomData.nextGaussian(0, 1));
    }
    double xbar = u.getMean();
    double s = u.getStandardDeviation();
    double n = u.getN();
    /*
     * t-test at .001-level TODO: replace with externalized t-test, with
     * test statistic defined in TestStatistic
     */
    assertTrue(FastMath.abs(xbar) / (s / FastMath.sqrt(n)) < 3.29);
}
 

/** test failure modes and distribution of nextGaussian() */
public void testNextGaussian() {
    try {
        randomData.nextGaussian(0, 0);
        fail("zero sigma -- IllegalArgumentException expected");
    } catch (IllegalArgumentException ex) {
        // ignored
    }
    SummaryStatistics u = new SummaryStatistics();
    for (int i = 0; i < largeSampleSize; i++) {
        u.addValue(randomData.nextGaussian(0, 1));
    }
    double xbar = u.getMean();
    double s = u.getStandardDeviation();
    double n = u.getN();
    /*
     * t-test at .001-level TODO: replace with externalized t-test, with
     * test statistic defined in TestStatistic
     */
    assertTrue(Math.abs(xbar) / (s / Math.sqrt(n)) < 3.29);
}
 

/**
 * Generates a random value from this distribution.
 * 
 * @return the random value.
 * @throws IllegalStateException if the distribution has not been loaded
 */
public double getNextValue() throws IllegalStateException {

    if (!loaded) {
        throw MathRuntimeException.createIllegalStateException("distribution not loaded");
    }

    // Start with a uniformly distributed random number in (0,1)
    double x = Math.random();

    // Use this to select the bin and generate a Gaussian within the bin
    for (int i = 0; i < binCount; i++) {
       if (x <= upperBounds[i]) {
           SummaryStatistics stats = binStats.get(i);
           if (stats.getN() > 0) {
               if (stats.getStandardDeviation() > 0) {  // more than one obs
                    return randomData.nextGaussian
                        (stats.getMean(),stats.getStandardDeviation());
               } else {
                   return stats.getMean(); // only one obs in bin
               }
           }
       }
    }
    throw new MathRuntimeException("no bin selected");
}
 

/** test failure modes and distribution of nextGaussian() */
public void testNextGaussian() {
    try {
        randomData.nextGaussian(0, 0);
        fail("zero sigma -- IllegalArgumentException expected");
    } catch (IllegalArgumentException ex) {
        // ignored
    }
    SummaryStatistics u = new SummaryStatistics();
    for (int i = 0; i < largeSampleSize; i++) {
        u.addValue(randomData.nextGaussian(0, 1));
    }
    double xbar = u.getMean();
    double s = u.getStandardDeviation();
    double n = u.getN();
    /*
     * t-test at .001-level TODO: replace with externalized t-test, with
     * test statistic defined in TestStatistic
     */
    assertTrue(Math.abs(xbar) / (s / Math.sqrt(n)) < 3.29);
}
 

/**
 * Test SummaryStatistics - implementations that do not store the data
 * and use single pass algorithms to compute statistics
*/
public void testSummaryStatistics() throws Exception {
    SummaryStatistics u = new SummaryStatistics();
    loadStats("data/PiDigits.txt", u);
    assertEquals("PiDigits: std", std, u.getStandardDeviation(), 1E-13);
    assertEquals("PiDigits: mean", mean, u.getMean(), 1E-13);

    loadStats("data/Mavro.txt", u);
    assertEquals("Mavro: std", std, u.getStandardDeviation(), 1E-14);
    assertEquals("Mavro: mean", mean, u.getMean(), 1E-14);

    loadStats("data/Michelso.txt", u);
    assertEquals("Michelso: std", std, u.getStandardDeviation(), 1E-13);
    assertEquals("Michelso: mean", mean, u.getMean(), 1E-13);

    loadStats("data/NumAcc1.txt", u);
    assertEquals("NumAcc1: std", std, u.getStandardDeviation(), 1E-14);
    assertEquals("NumAcc1: mean", mean, u.getMean(), 1E-14);

    loadStats("data/NumAcc2.txt", u);
    assertEquals("NumAcc2: std", std, u.getStandardDeviation(), 1E-14);
    assertEquals("NumAcc2: mean", mean, u.getMean(), 1E-14);
}
 

/**
 * Test SummaryStatistics - implementations that do not store the data
 * and use single pass algorithms to compute statistics
*/
public void testSummaryStatistics() throws Exception {
    SummaryStatistics u = new SummaryStatistics();
    loadStats("data/PiDigits.txt", u);
    assertEquals("PiDigits: std", std, u.getStandardDeviation(), 1E-13);
    assertEquals("PiDigits: mean", mean, u.getMean(), 1E-13);  

    loadStats("data/Mavro.txt", u);
    assertEquals("Mavro: std", std, u.getStandardDeviation(), 1E-14);
    assertEquals("Mavro: mean", mean, u.getMean(), 1E-14);
    
    loadStats("data/Michelso.txt", u);
    assertEquals("Michelso: std", std, u.getStandardDeviation(), 1E-13);
    assertEquals("Michelso: mean", mean, u.getMean(), 1E-13);   
                                    
    loadStats("data/NumAcc1.txt", u);
    assertEquals("NumAcc1: std", std, u.getStandardDeviation(), 1E-14);
    assertEquals("NumAcc1: mean", mean, u.getMean(), 1E-14);
    
    loadStats("data/NumAcc2.txt", u);
    assertEquals("NumAcc2: std", std, u.getStandardDeviation(), 1E-14);
    assertEquals("NumAcc2: mean", mean, u.getMean(), 1E-14);
}
 

/**
 * Generates a random value from this distribution.
 *
 * @return the random value.
 * @throws IllegalStateException if the distribution has not been loaded
 */
public double getNextValue() throws IllegalStateException {

    if (!loaded) {
        throw MathRuntimeException.createIllegalStateException("distribution not loaded");
    }

    // Start with a uniformly distributed random number in (0,1)
    double x = Math.random();

    // Use this to select the bin and generate a Gaussian within the bin
    for (int i = 0; i < binCount; i++) {
       if (x <= upperBounds[i]) {
           SummaryStatistics stats = binStats.get(i);
           if (stats.getN() > 0) {
               if (stats.getStandardDeviation() > 0) {  // more than one obs
                    return randomData.nextGaussian
                        (stats.getMean(),stats.getStandardDeviation());
               } else {
                   return stats.getMean(); // only one obs in bin
               }
           }
       }
    }
    throw new MathRuntimeException("no bin selected");
}
 

/**
 * Generates a random value from this distribution.
 *
 * @return the random value.
 * @throws IllegalStateException if the distribution has not been loaded
 */
public double getNextValue() throws IllegalStateException {

    if (!loaded) {
        throw MathRuntimeException.createIllegalStateException(LocalizedFormats.DISTRIBUTION_NOT_LOADED);
    }

    // Start with a uniformly distributed random number in (0,1)
    double x = FastMath.random();

    // Use this to select the bin and generate a Gaussian within the bin
    for (int i = 0; i < binCount; i++) {
       if (x <= upperBounds[i]) {
           SummaryStatistics stats = binStats.get(i);
           if (stats.getN() > 0) {
               if (stats.getStandardDeviation() > 0) {  // more than one obs
                    return randomData.nextGaussian
                        (stats.getMean(),stats.getStandardDeviation());
               } else {
                   return stats.getMean(); // only one obs in bin
               }
           }
       }
    }
    throw new MathRuntimeException(LocalizedFormats.NO_BIN_SELECTED);
}
 

/** test failure modes and distribution of nextGaussian() */
@Test
public void testNextGaussian() {
    try {
        randomData.nextGaussian(0, 0);
        Assert.fail("zero sigma -- MathIllegalArgumentException expected");
    } catch (MathIllegalArgumentException ex) {
        // ignored
    }
    SummaryStatistics u = new SummaryStatistics();
    for (int i = 0; i < largeSampleSize; i++) {
        u.addValue(randomData.nextGaussian(0, 1));
    }
    double xbar = u.getMean();
    double s = u.getStandardDeviation();
    double n = u.getN();
    /*
     * t-test at .001-level TODO: replace with externalized t-test, with
     * test statistic defined in TestStatistic
     */
    Assert.assertTrue(FastMath.abs(xbar) / (s / FastMath.sqrt(n)) < 3.29);
}
 

/**
 * Generates a random value from this distribution.
 *
 * @return the random value.
 * @throws IllegalStateException if the distribution has not been loaded
 */
public double getNextValue() throws IllegalStateException {

    if (!loaded) {
        throw MathRuntimeException.createIllegalStateException(LocalizedFormats.DISTRIBUTION_NOT_LOADED);
    }

    // Start with a uniformly distributed random number in (0,1)
    double x = FastMath.random();

    // Use this to select the bin and generate a Gaussian within the bin
    for (int i = 0; i < binCount; i++) {
       if (x <= upperBounds[i]) {
           SummaryStatistics stats = binStats.get(i);
           if (stats.getN() > 0) {
               if (stats.getStandardDeviation() > 0) {  // more than one obs
                    return randomData.nextGaussian
                        (stats.getMean(),stats.getStandardDeviation());
               } else {
                   return stats.getMean(); // only one obs in bin
               }
           }
       }
    }
    throw new MathRuntimeException(LocalizedFormats.NO_BIN_SELECTED);
}
 

@Test
public void     testRandom() throws Exception
{
    final int                       QTY = 10;
    final int                       ITERATIONS = 1000;

    TestInstanceProvider            instanceProvider = new TestInstanceProvider(QTY, 0);
    ProviderStrategy<Void>          strategy = new RandomStrategy<Void>();

    long[]                          counts = new long[QTY];
    for ( int i = 0; i < ITERATIONS; ++i )
    {
        ServiceInstance<Void>   instance = strategy.getInstance(instanceProvider);
        int                     id = Integer.parseInt(instance.getId());
        counts[id]++;
    }

    SummaryStatistics               statistic = new SummaryStatistics();
    for ( int i = 0; i < QTY; ++i )
    {
        statistic.addValue(counts[i]);
    }
    Assert.assertTrue(statistic.getStandardDeviation() <= (QTY * 2), "" + statistic.getStandardDeviation()); // meager check for even distribution
}
 

/** test failure modes and distribution of nextGaussian() */  
public void testNextGaussian() { 
    try {
        double x = randomData.nextGaussian(0,0);
        fail("zero sigma -- IllegalArgumentException expected");
    } catch (IllegalArgumentException ex) {
        ;
    }
    SummaryStatistics u = SummaryStatistics.newInstance();
    for (int i = 0; i<largeSampleSize; i++) {
        u.addValue(randomData.nextGaussian(0,1));
    }
    double xbar = u.getMean();
    double s = u.getStandardDeviation();
    double n = (double) u.getN(); 
    /* t-test at .001-level TODO: replace with externalized t-test, with
     * test statistic defined in TestStatistic
     */
    assertTrue(Math.abs(xbar)/(s/Math.sqrt(n))< 3.29);
}
 

/**
 * Test SummaryStatistics - implementations that do not store the data
 * and use single pass algorithms to compute statistics
*/
public void testSummaryStatistics() throws Exception {
    SummaryStatistics u = new SummaryStatistics();
    loadStats("data/PiDigits.txt", u);
    assertEquals("PiDigits: std", std, u.getStandardDeviation(), 1E-13);
    assertEquals("PiDigits: mean", mean, u.getMean(), 1E-13);

    loadStats("data/Mavro.txt", u);
    assertEquals("Mavro: std", std, u.getStandardDeviation(), 1E-14);
    assertEquals("Mavro: mean", mean, u.getMean(), 1E-14);

    loadStats("data/Michelso.txt", u);
    assertEquals("Michelso: std", std, u.getStandardDeviation(), 1E-13);
    assertEquals("Michelso: mean", mean, u.getMean(), 1E-13);

    loadStats("data/NumAcc1.txt", u);
    assertEquals("NumAcc1: std", std, u.getStandardDeviation(), 1E-14);
    assertEquals("NumAcc1: mean", mean, u.getMean(), 1E-14);

    loadStats("data/NumAcc2.txt", u);
    assertEquals("NumAcc2: std", std, u.getStandardDeviation(), 1E-14);
    assertEquals("NumAcc2: mean", mean, u.getMean(), 1E-14);
}
 

/**
 * Test SummaryStatistics - implementations that do not store the data
 * and use single pass algorithms to compute statistics
*/
public void testSummaryStatistics() throws Exception {
    SummaryStatistics u = new SummaryStatistics();
    loadStats("data/PiDigits.txt", u);
    assertEquals("PiDigits: std", std, u.getStandardDeviation(), 1E-13);
    assertEquals("PiDigits: mean", mean, u.getMean(), 1E-13);  

    loadStats("data/Mavro.txt", u);
    assertEquals("Mavro: std", std, u.getStandardDeviation(), 1E-14);
    assertEquals("Mavro: mean", mean, u.getMean(), 1E-14);
    
    loadStats("data/Michelso.txt", u);
    assertEquals("Michelso: std", std, u.getStandardDeviation(), 1E-13);
    assertEquals("Michelso: mean", mean, u.getMean(), 1E-13);   
                                    
    loadStats("data/NumAcc1.txt", u);
    assertEquals("NumAcc1: std", std, u.getStandardDeviation(), 1E-14);
    assertEquals("NumAcc1: mean", mean, u.getMean(), 1E-14);
    
    loadStats("data/NumAcc2.txt", u);
    assertEquals("NumAcc2: std", std, u.getStandardDeviation(), 1E-14);
    assertEquals("NumAcc2: mean", mean, u.getMean(), 1E-14);
}
 

/** test failure modes and distribution of nextGaussian() */
public void testNextGaussian() {
    try {
        randomData.nextGaussian(0, 0);
        fail("zero sigma -- IllegalArgumentException expected");
    } catch (IllegalArgumentException ex) {
        // ignored
    }
    SummaryStatistics u = new SummaryStatistics();
    for (int i = 0; i < largeSampleSize; i++) {
        u.addValue(randomData.nextGaussian(0, 1));
    }
    double xbar = u.getMean();
    double s = u.getStandardDeviation();
    double n = u.getN();
    /*
     * t-test at .001-level TODO: replace with externalized t-test, with
     * test statistic defined in TestStatistic
     */
    assertTrue(Math.abs(xbar) / (s / Math.sqrt(n)) < 3.29);
}
 

/** test failure modes and distribution of nextGaussian() */
public void testNextGaussian() {
	try {
		randomData.nextGaussian(0, 0);
		fail("zero sigma -- IllegalArgumentException expected");
	} catch (IllegalArgumentException ex) {
		// ignored
	}
	SummaryStatistics u = new SummaryStatistics();
	for (int i = 0; i < largeSampleSize; i++) {
		u.addValue(randomData.nextGaussian(0, 1));
	}
	double xbar = u.getMean();
	double s = u.getStandardDeviation();
	double n = u.getN();
	/*
	 * t-test at .001-level TODO: replace with externalized t-test, with
	 * test statistic defined in TestStatistic
	 */
	assertTrue(Math.abs(xbar) / (s / Math.sqrt(n)) < 3.29);
}
 

/** test failure modes and distribution of nextGaussian() */
public void testNextGaussian() {
    try {
        randomData.nextGaussian(0, 0);
        fail("zero sigma -- IllegalArgumentException expected");
    } catch (IllegalArgumentException ex) {
        // ignored
    }
    SummaryStatistics u = new SummaryStatistics();
    for (int i = 0; i < largeSampleSize; i++) {
        u.addValue(randomData.nextGaussian(0, 1));
    }
    double xbar = u.getMean();
    double s = u.getStandardDeviation();
    double n = u.getN();
    /*
     * t-test at .001-level TODO: replace with externalized t-test, with
     * test statistic defined in TestStatistic
     */
    assertTrue(Math.abs(xbar) / (s / Math.sqrt(n)) < 3.29);
}
 

private void tstGen(double tolerance)throws Exception {
    empiricalDistribution.load(url);
    SummaryStatistics stats = new SummaryStatistics();
    for (int i = 1; i < 1000; i++) {
        stats.addValue(empiricalDistribution.getNextValue());
    }
    assertEquals("mean", stats.getMean(),5.069831575018909,tolerance);
    assertEquals
     ("std dev", stats.getStandardDeviation(),1.0173699343977738,tolerance);
}
 

/** {@inheritDoc} */
@Override
public void computeBinStats() throws IOException {
    for (int i = 0; i < inputArray.length; i++) {
        SummaryStatistics stats =
            binStats.get(findBin(inputArray[i]));
        stats.addValue(inputArray[i]);
    }
}
 

private void tstDoubleGen(double tolerance)throws Exception {
    empiricalDistribution2.load(dataArray);
    SummaryStatistics stats = new SummaryStatistics();
    for (int i = 1; i < 1000; i++) {
        stats.addValue(empiricalDistribution2.getNextValue());
    }
    assertEquals("mean", stats.getMean(),5.069831575018909,tolerance);
    assertEquals
     ("std dev", stats.getStandardDeviation(),1.0173699343977738,tolerance);
}
 

@Test
public void testInternalDifferentiation()
    throws IntegratorException, DerivativeException {
    FirstOrderIntegrator integ =
        new DormandPrince54Integrator(1.0e-8, 100.0, new double[] { 1.0e-4, 1.0e-4 }, new double[] { 1.0e-4, 1.0e-4 });
    double hP = 1.0e-12;
    SummaryStatistics residualsP0 = new SummaryStatistics();
    SummaryStatistics residualsP1 = new SummaryStatistics();
    for (double b = 2.88; b < 3.08; b += 0.001) {
        Brusselator brusselator = new Brusselator(b);
        brusselator.setParameter(0, b);
        double[] z = { 1.3, b };
        double[][] dZdZ0 = new double[2][2];
        double[][] dZdP  = new double[2][1];
        double hY = 1.0e-12;
        FirstOrderIntegratorWithJacobians extInt =
            new FirstOrderIntegratorWithJacobians(integ, brusselator, new double[] { b },
                                                  new double[] { hY, hY }, new double[] { hP });
        extInt.setMaxEvaluations(5000);
        extInt.integrate(0, z, new double[][] { { 0.0 }, { 1.0 } }, 20.0, z, dZdZ0, dZdP);
        Assert.assertEquals(5000, extInt.getMaxEvaluations());
        Assert.assertTrue(extInt.getEvaluations() > 1500);
        Assert.assertTrue(extInt.getEvaluations() < 2100);
        Assert.assertEquals(4 * integ.getEvaluations(), extInt.getEvaluations());
        residualsP0.addValue(dZdP[0][0] - brusselator.dYdP0());
        residualsP1.addValue(dZdP[1][0] - brusselator.dYdP1());
    }
    Assert.assertTrue((residualsP0.getMax() - residualsP0.getMin()) < 0.02);
    Assert.assertTrue(residualsP0.getStandardDeviation() < 0.003);
    Assert.assertTrue((residualsP1.getMax() - residualsP1.getMin()) < 0.05);
    Assert.assertTrue(residualsP1.getStandardDeviation() < 0.01);
}
 

/**
 * Fills binStats array (second pass through data file).
 *
 * @param in object providing access to the data
 * @throws IOException  if an IO error occurs
 */
private void fillBinStats(Object in) throws IOException {
    // Set up grid
    min = sampleStats.getMin();
    max = sampleStats.getMax();
    delta = (max - min)/(Double.valueOf(binCount)).doubleValue();

    // Initialize binStats ArrayList
    if (!binStats.isEmpty()) {
        binStats.clear();
    }
    for (int i = 0; i < binCount; i++) {
        SummaryStatistics stats = new SummaryStatistics();
        binStats.add(i,stats);
    }

    // Filling data in binStats Array
    DataAdapterFactory aFactory = new DataAdapterFactory();
    DataAdapter da = aFactory.getAdapter(in);
    da.computeBinStats();

    // Assign upperBounds based on bin counts
    upperBounds = new double[binCount];
    upperBounds[0] =
    ((double) binStats.get(0).getN()) / (double) sampleStats.getN();
    for (int i = 1; i < binCount-1; i++) {
        upperBounds[i] = upperBounds[i-1] +
        ((double) binStats.get(i).getN()) / (double) sampleStats.getN();
    }
    upperBounds[binCount-1] = 1.0d;
}
 

/**
 * Computes binStats
 * 
 * @param min  minimum value
 * @param delta  grid size
 * @throws IOException  if an IO error occurs
 */
@Override
public void computeBinStats(double min, double delta)
    throws IOException {
    for (int i = 0; i < inputArray.length; i++) {
        SummaryStatistics stats =
            binStats.get(findBin(min, inputArray[i], delta));
        stats.addValue(inputArray[i]);
    }
}
 

/** {@inheritDoc} */
@Override
public void computeStats() throws IOException {
    sampleStats = new SummaryStatistics();
    for (int i = 0; i < inputArray.length; i++) {
        sampleStats.addValue(inputArray[i]);
    }
}
 

/** {@inheritDoc} */
@Override
public void computeStats() throws IOException {
    sampleStats = new SummaryStatistics();
    for (int i = 0; i < inputArray.length; i++) {
        sampleStats.addValue(inputArray[i]);
    }
}
 

/** {@inheritDoc} */
@Override
public void computeBinStats() throws IOException {
    String str = null;
    double val = 0.0d;
    while ((str = inputStream.readLine()) != null) {
        val = Double.parseDouble(str);
        SummaryStatistics stats = binStats.get(findBin(val));
        stats.addValue(val);
    }

    inputStream.close();
    inputStream = null;
}
 

/** {@inheritDoc} */
@Override
public void computeBinStats() throws IOException {
    for (int i = 0; i < inputArray.length; i++) {
        SummaryStatistics stats =
            binStats.get(findBin(inputArray[i]));
        stats.addValue(inputArray[i]);
    }
}
 

@Test
public void testGaussian() {
    MersenneTwister mt = new MersenneTwister(42853252100l);
    SummaryStatistics sample = new SummaryStatistics();
    for (int i = 0; i < 1000; ++i) {
        sample.addValue(mt.nextGaussian());
    }
    assertEquals(0.0, sample.getMean(), 0.005);
    assertEquals(1.0, sample.getStandardDeviation(), 0.025);
}