org.apache.commons.math.stat.descriptive.DescriptiveStatistics Java Examples

/**
 * Normalize (standardize) the series, so in the end it is having a mean of 0 and a standard deviation of 1.
 *
 * @param sample Sample to normalize.
 * @return normalized (standardized) sample.
 * @since 2.2
 */
public static double[] normalize(final double[] sample) {
    DescriptiveStatistics stats = new DescriptiveStatistics();

    // Add the data from the series to stats
    for (int i = 0; i < sample.length; i++) {
        stats.addValue(sample[i]);
    }

    // Compute mean and standard deviation
    double mean = stats.getMean();
    double standardDeviation = stats.getStandardDeviation();

    // initialize the standardizedSample, which has the same length as the sample
    double[] standardizedSample = new double[sample.length];

    for (int i = 0; i < sample.length; i++) {
        // z = (x- mean)/standardDeviation
        standardizedSample[i] = (sample[i] - mean) / standardDeviation;
    }
    return standardizedSample;
}
 

/** {@inheritDoc} */
@Override
public double[] getValues() {

    int length = list.size();

    // If the window size is not INFINITE_WINDOW AND
    // the current list is larger that the window size, we need to
    // take into account only the last n elements of the list
    // as definied by windowSize

    if (windowSize != DescriptiveStatistics.INFINITE_WINDOW &&
        windowSize < list.size())
    {
        length = list.size() - Math.max(0, list.size() - windowSize);
    }

    // Create an array to hold all values
    double[] copiedArray = new double[length];

    for (int i = 0; i < copiedArray.length; i++) {
        copiedArray[i] = getElement(i);
    }
    return copiedArray;
}
 

/** {@inheritDoc} */
@Override
public long getN() {
    int n = 0;

    if (windowSize != DescriptiveStatistics.INFINITE_WINDOW) {
        if (list.size() > windowSize) {
            n = windowSize;
        } else {
            n = list.size();
        }
    } else {
        n = list.size();
    }
    return n;
}
 

public void conductExperiment(final MySQLReductionExperiment exp) throws Exception {
	List<Map<String, Object>> mccvResults = Util.conductSingleOneStepReductionExperiment(exp.getExperiment());
	DescriptiveStatistics errorRate = new DescriptiveStatistics();
	DescriptiveStatistics runtime = new DescriptiveStatistics();
	for (Map<String, Object> result : mccvResults) {
		errorRate.addValue((double) result.get("errorRate"));
		runtime.addValue((long) result.get("trainTime"));
	}

	/* prepapre values for experiment update */
	Map<String, Object> values = new HashMap<>();
	values.put(ERROR_RATE_MIN, errorRate.getMin());
	values.put(ERROR_RATE_MAX, errorRate.getMax());
	values.put(ERROR_RATE_MEAN, errorRate.getMean());
	values.put(ERROR_RATE_STD, errorRate.getStandardDeviation());
	values.put(RUNTIME_MIN, runtime.getMin());
	values.put(RUNTIME_MAX, runtime.getMax());
	values.put(RUNTIME_MEAN, runtime.getMean());
	values.put(RUNTIME_STD, runtime.getStandardDeviation());
	this.updateExperiment(exp, values);
}
 

/** {@inheritDoc} */
@Override
public long getN() {
    int n = 0;

    if (windowSize != DescriptiveStatistics.INFINITE_WINDOW) {
        if (list.size() > windowSize) {
            n = windowSize;
        } else {
            n = list.size();
        }
    } else {
        n = list.size();
    }
    return n;
}
 

/**
 * Test DescriptiveStatistics - implementations that store full array of
 * values and execute multi-pass algorithms
 */
public void testDescriptiveStatistics() throws Exception {

    DescriptiveStatistics u = new DescriptiveStatistics();
    
    loadStats("data/PiDigits.txt", u);
    assertEquals("PiDigits: std", std, u.getStandardDeviation(), 1E-14);
    assertEquals("PiDigits: mean", mean, u.getMean(), 1E-14);
    
    loadStats("data/Mavro.txt", u);
    assertEquals("Mavro: std", std, u.getStandardDeviation(), 1E-14);
    assertEquals("Mavro: mean", mean, u.getMean(), 1E-14);        
    
    loadStats("data/Michelso.txt", u);
    assertEquals("Michelso: std", std, u.getStandardDeviation(), 1E-14);
    assertEquals("Michelso: mean", mean, u.getMean(), 1E-14);   

    loadStats("data/NumAcc1.txt", u);
    assertEquals("NumAcc1: std", std, u.getStandardDeviation(), 1E-14);
    assertEquals("NumAcc1: mean", mean, u.getMean(), 1E-14);
    
    loadStats("data/NumAcc2.txt", u);
    assertEquals("NumAcc2: std", std, u.getStandardDeviation(), 1E-14);
    assertEquals("NumAcc2: mean", mean, u.getMean(), 1E-14);
}
 

/** {@inheritDoc} */
@Override
public double[] getValues() {

    int length = list.size();

    // If the window size is not INFINITE_WINDOW AND
    // the current list is larger that the window size, we need to
    // take into account only the last n elements of the list
    // as definied by windowSize

    if (windowSize != DescriptiveStatistics.INFINITE_WINDOW &&
        windowSize < list.size())
    {
        length = list.size() - Math.max(0, list.size() - windowSize);
    }

    // Create an array to hold all values
    double[] copiedArray = new double[length];

    for (int i = 0; i < copiedArray.length; i++) {
        copiedArray[i] = getElement(i);
    }
    return copiedArray;
}
 

/** {@inheritDoc} */
@Override
public double[] getValues() {

    int length = list.size();

    // If the window size is not INFINITE_WINDOW AND
    // the current list is larger that the window size, we need to
    // take into account only the last n elements of the list
    // as definied by windowSize

    if (windowSize != DescriptiveStatistics.INFINITE_WINDOW &&
        windowSize < list.size())
    {
        length = list.size() - Math.max(0, list.size() - windowSize);
    }

    // Create an array to hold all values
    double[] copiedArray = new double[length];

    for (int i = 0; i < copiedArray.length; i++) {
        copiedArray[i] = getElement(i);
    }
    return copiedArray;
}
 

/** {@inheritDoc} */
@Override
public double getElement(int index) {

    double value = Double.NaN;

    int calcIndex = index;

    if (windowSize != DescriptiveStatistics.INFINITE_WINDOW &&
        windowSize < list.size())
    {
        calcIndex = (list.size() - windowSize) + index;
    }


    try {
        value = transformer.transform(list.get(calcIndex));
    } catch (MathException e) {
        e.printStackTrace();
    }

    return value;
}
 

/** {@inheritDoc} */
@Override
public long getN() {
    int n = 0;

    if (windowSize != DescriptiveStatistics.INFINITE_WINDOW) {
        if (list.size() > windowSize) {
            n = windowSize;
        } else {
            n = list.size();
        }
    } else {
        n = list.size();
    }
    return n;
}
 

/** {@inheritDoc} */
@Override
public double[] getValues() {

    int length = list.size();

    // If the window size is not INFINITE_WINDOW AND
    // the current list is larger that the window size, we need to
    // take into account only the last n elements of the list
    // as definied by windowSize

    if (windowSize != DescriptiveStatistics.INFINITE_WINDOW &&
        windowSize < list.size())
    {
        length = list.size() - FastMath.max(0, list.size() - windowSize);
    }

    // Create an array to hold all values
    double[] copiedArray = new double[length];

    for (int i = 0; i < copiedArray.length; i++) {
        copiedArray[i] = getElement(i);
    }
    return copiedArray;
}
 

/**
 * Test DescriptiveStatistics - implementations that store full array of
 * values and execute multi-pass algorithms
 */
public void testDescriptiveStatistics() throws Exception {

    DescriptiveStatistics u = new DescriptiveStatistics();

    loadStats("data/PiDigits.txt", u);
    assertEquals("PiDigits: std", std, u.getStandardDeviation(), 1E-14);
    assertEquals("PiDigits: mean", mean, u.getMean(), 1E-14);

    loadStats("data/Mavro.txt", u);
    assertEquals("Mavro: std", std, u.getStandardDeviation(), 1E-14);
    assertEquals("Mavro: mean", mean, u.getMean(), 1E-14);

    loadStats("data/Michelso.txt", u);
    assertEquals("Michelso: std", std, u.getStandardDeviation(), 1E-14);
    assertEquals("Michelso: mean", mean, u.getMean(), 1E-14);

    loadStats("data/NumAcc1.txt", u);
    assertEquals("NumAcc1: std", std, u.getStandardDeviation(), 1E-14);
    assertEquals("NumAcc1: mean", mean, u.getMean(), 1E-14);

    loadStats("data/NumAcc2.txt", u);
    assertEquals("NumAcc2: std", std, u.getStandardDeviation(), 1E-14);
    assertEquals("NumAcc2: mean", mean, u.getMean(), 1E-14);
}
 

/**
 * Test StorelessDescriptiveStatistics
 */
public void testStoredUnivariateImpl() throws Exception {

    DescriptiveStatistics u = DescriptiveStatistics.newInstance();
    
    loadStats("data/PiDigits.txt", u);
    assertEquals("PiDigits: std", std, u.getStandardDeviation(), .0000000000001);
    assertEquals("PiDigits: mean", mean, u.getMean(), .0000000000001);
    
    loadStats("data/Mavro.txt", u);
    assertEquals("Mavro: std", std, u.getStandardDeviation(), .00000000000001);
    assertEquals("Mavro: mean", mean, u.getMean(), .00000000000001);        
    
    //loadStats("data/Michelso.txt");
    //assertEquals("Michelso: std", std, u.getStandardDeviation(), .00000000000001);
    //assertEquals("Michelso: mean", mean, u.getMean(), .00000000000001);   

    loadStats("data/NumAcc1.txt", u);
    assertEquals("NumAcc1: std", std, u.getStandardDeviation(), .00000000000001);
    assertEquals("NumAcc1: mean", mean, u.getMean(), .00000000000001);
    
    //loadStats("data/NumAcc2.txt");
    //assertEquals("NumAcc2: std", std, u.getStandardDeviation(), .000000001);
    //assertEquals("NumAcc2: mean", mean, u.getMean(), .00000000000001);
}
 

/**
 * Test DescriptiveStatistics - implementations that store full array of
 * values and execute multi-pass algorithms
 */
public void testDescriptiveStatistics() throws Exception {

    DescriptiveStatistics u = new DescriptiveStatistics();

    loadStats("data/PiDigits.txt", u);
    assertEquals("PiDigits: std", std, u.getStandardDeviation(), 1E-14);
    assertEquals("PiDigits: mean", mean, u.getMean(), 1E-14);

    loadStats("data/Mavro.txt", u);
    assertEquals("Mavro: std", std, u.getStandardDeviation(), 1E-14);
    assertEquals("Mavro: mean", mean, u.getMean(), 1E-14);

    loadStats("data/Michelso.txt", u);
    assertEquals("Michelso: std", std, u.getStandardDeviation(), 1E-14);
    assertEquals("Michelso: mean", mean, u.getMean(), 1E-14);

    loadStats("data/NumAcc1.txt", u);
    assertEquals("NumAcc1: std", std, u.getStandardDeviation(), 1E-14);
    assertEquals("NumAcc1: mean", mean, u.getMean(), 1E-14);

    loadStats("data/NumAcc2.txt", u);
    assertEquals("NumAcc2: std", std, u.getStandardDeviation(), 1E-14);
    assertEquals("NumAcc2: mean", mean, u.getMean(), 1E-14);
}
 

/**
 * Test DescriptiveStatistics - implementations that store full array of
 * values and execute multi-pass algorithms
 */
public void testDescriptiveStatistics() throws Exception {

    DescriptiveStatistics u = new DescriptiveStatistics();

    loadStats("data/PiDigits.txt", u);
    assertEquals("PiDigits: std", std, u.getStandardDeviation(), 1E-14);
    assertEquals("PiDigits: mean", mean, u.getMean(), 1E-14);

    loadStats("data/Mavro.txt", u);
    assertEquals("Mavro: std", std, u.getStandardDeviation(), 1E-14);
    assertEquals("Mavro: mean", mean, u.getMean(), 1E-14);

    loadStats("data/Michelso.txt", u);
    assertEquals("Michelso: std", std, u.getStandardDeviation(), 1E-14);
    assertEquals("Michelso: mean", mean, u.getMean(), 1E-14);

    loadStats("data/NumAcc1.txt", u);
    assertEquals("NumAcc1: std", std, u.getStandardDeviation(), 1E-14);
    assertEquals("NumAcc1: mean", mean, u.getMean(), 1E-14);

    loadStats("data/NumAcc2.txt", u);
    assertEquals("NumAcc2: std", std, u.getStandardDeviation(), 1E-14);
    assertEquals("NumAcc2: mean", mean, u.getMean(), 1E-14);
}
 

/**
 * Test DescriptiveStatistics - implementations that store full array of
 * values and execute multi-pass algorithms
 */
public void testDescriptiveStatistics() throws Exception {

    DescriptiveStatistics u = new DescriptiveStatistics();

    loadStats("data/PiDigits.txt", u);
    assertEquals("PiDigits: std", std, u.getStandardDeviation(), 1E-14);
    assertEquals("PiDigits: mean", mean, u.getMean(), 1E-14);

    loadStats("data/Mavro.txt", u);
    assertEquals("Mavro: std", std, u.getStandardDeviation(), 1E-14);
    assertEquals("Mavro: mean", mean, u.getMean(), 1E-14);

    loadStats("data/Michelso.txt", u);
    assertEquals("Michelso: std", std, u.getStandardDeviation(), 1E-14);
    assertEquals("Michelso: mean", mean, u.getMean(), 1E-14);

    loadStats("data/NumAcc1.txt", u);
    assertEquals("NumAcc1: std", std, u.getStandardDeviation(), 1E-14);
    assertEquals("NumAcc1: mean", mean, u.getMean(), 1E-14);

    loadStats("data/NumAcc2.txt", u);
    assertEquals("NumAcc2: std", std, u.getStandardDeviation(), 1E-14);
    assertEquals("NumAcc2: mean", mean, u.getMean(), 1E-14);
}
 

private void scale(double[][] peakList) {
  DescriptiveStatistics stdDevStats = new DescriptiveStatistics();

  for (int columns = 0; columns < peakList.length; columns++) {
    stdDevStats.clear();
    for (int row = 0; row < peakList[columns].length; row++) {
      if (!Double.isInfinite(peakList[columns][row]) && !Double.isNaN(peakList[columns][row])) {
        stdDevStats.addValue(peakList[columns][row]);
      }
    }

    double stdDev = stdDevStats.getStandardDeviation();

    for (int row = 0; row < peakList[columns].length; row++) {
      if (stdDev != 0) {
        peakList[columns][row] = peakList[columns][row] / stdDev;
      }
    }
  }
}
 

/** {@inheritDoc} */
@Override
public long getN() {
    int n = 0;

    if (windowSize != DescriptiveStatistics.INFINITE_WINDOW) {
        if (list.size() > windowSize) {
            n = windowSize;
        } else {
            n = list.size();
        }
    } else {
        n = list.size();
    }
    return n;
}
 

/** {@inheritDoc} */
@Override
public double getElement(int index) {

    double value = Double.NaN;

    int calcIndex = index;

    if (windowSize != DescriptiveStatistics.INFINITE_WINDOW &&
        windowSize < list.size())
    {
        calcIndex = (list.size() - windowSize) + index;
    }

    
    try {
        value = transformer.transform(list.get(calcIndex));
    } catch (MathException e) {
        e.printStackTrace();
    }
    
    return value;
}
 

/** {@inheritDoc} */
@Override
public double getElement(int index) {

    double value = Double.NaN;

    int calcIndex = index;

    if (windowSize != DescriptiveStatistics.INFINITE_WINDOW &&
        windowSize < list.size())
    {
        calcIndex = (list.size() - windowSize) + index;
    }


    try {
        value = transformer.transform(list.get(calcIndex));
    } catch (MathException e) {
        e.printStackTrace();
    }

    return value;
}
 

/** {@inheritDoc} */
@Override
public double[] getValues() {

    int length = list.size();

    // If the window size is not INFINITE_WINDOW AND
    // the current list is larger that the window size, we need to
    // take into account only the last n elements of the list
    // as definied by windowSize

    if (windowSize != DescriptiveStatistics.INFINITE_WINDOW &&
        windowSize < list.size())
    {
        length = list.size() - Math.max(0, list.size() - windowSize);
    }

    // Create an array to hold all values
    double[] copiedArray = new double[length];

    for (int i = 0; i < copiedArray.length; i++) {
        copiedArray[i] = getElement(i);
    }
    return copiedArray;
}
 

/** {@inheritDoc} */
@Override
public double[] getValues() {

    int length = list.size();

    // If the window size is not INFINITE_WINDOW AND
    // the current list is larger that the window size, we need to
    // take into account only the last n elements of the list
    // as definied by windowSize

    if (windowSize != DescriptiveStatistics.INFINITE_WINDOW &&
        windowSize < list.size())
    {
        length = list.size() - FastMath.max(0, list.size() - windowSize);
    }

    // Create an array to hold all values
    double[] copiedArray = new double[length];

    for (int i = 0; i < copiedArray.length; i++) {
        copiedArray[i] = getElement(i);
    }
    return copiedArray;
}
 

/** {@inheritDoc} */
@Override
public double getElement(int index) {

    double value = Double.NaN;

    int calcIndex = index;

    if (windowSize != DescriptiveStatistics.INFINITE_WINDOW &&
        windowSize < list.size())
    {
        calcIndex = (list.size() - windowSize) + index;
    }


    try {
        value = transformer.transform(list.get(calcIndex));
    } catch (MathException e) {
        e.printStackTrace();
    }

    return value;
}
 

/** {@inheritDoc} */
@Override
public long getN() {
    int n = 0;

    if (windowSize != DescriptiveStatistics.INFINITE_WINDOW) {
        if (list.size() > windowSize) {
            n = windowSize;
        } else {
            n = list.size();
        }
    } else {
        n = list.size();
    }
    return n;
}
 

/** {@inheritDoc} */
@Override
public double getElement(int index) {

    double value = Double.NaN;

    int calcIndex = index;

    if (windowSize != DescriptiveStatistics.INFINITE_WINDOW &&
        windowSize < list.size())
    {
        calcIndex = (list.size() - windowSize) + index;
    }

    
    try {
        value = transformer.transform(list.get(calcIndex));
    } catch (MathException e) {
        e.printStackTrace();
    }
    
    return value;
}
 

/**
 * Test DescriptiveStatistics - implementations that store full array of
 * values and execute multi-pass algorithms
 */
@Test
public void testDescriptiveStatistics() throws Exception {

    DescriptiveStatistics u = new DescriptiveStatistics();

    loadStats("data/PiDigits.txt", u);
    Assert.assertEquals("PiDigits: std", std, u.getStandardDeviation(), 1E-14);
    Assert.assertEquals("PiDigits: mean", mean, u.getMean(), 1E-14);

    loadStats("data/Mavro.txt", u);
    Assert.assertEquals("Mavro: std", std, u.getStandardDeviation(), 1E-14);
    Assert.assertEquals("Mavro: mean", mean, u.getMean(), 1E-14);

    loadStats("data/Michelso.txt", u);
    Assert.assertEquals("Michelso: std", std, u.getStandardDeviation(), 1E-14);
    Assert.assertEquals("Michelso: mean", mean, u.getMean(), 1E-14);

    loadStats("data/NumAcc1.txt", u);
    Assert.assertEquals("NumAcc1: std", std, u.getStandardDeviation(), 1E-14);
    Assert.assertEquals("NumAcc1: mean", mean, u.getMean(), 1E-14);

    loadStats("data/NumAcc2.txt", u);
    Assert.assertEquals("NumAcc2: std", std, u.getStandardDeviation(), 1E-14);
    Assert.assertEquals("NumAcc2: mean", mean, u.getMean(), 1E-14);
}
 

/**
 * Test DescriptiveStatistics - implementations that store full array of
 * values and execute multi-pass algorithms
 */
public void testDescriptiveStatistics() throws Exception {

    DescriptiveStatistics u = new DescriptiveStatistics();
    
    loadStats("data/PiDigits.txt", u);
    assertEquals("PiDigits: std", std, u.getStandardDeviation(), 1E-14);
    assertEquals("PiDigits: mean", mean, u.getMean(), 1E-14);
    
    loadStats("data/Mavro.txt", u);
    assertEquals("Mavro: std", std, u.getStandardDeviation(), 1E-14);
    assertEquals("Mavro: mean", mean, u.getMean(), 1E-14);        
    
    loadStats("data/Michelso.txt", u);
    assertEquals("Michelso: std", std, u.getStandardDeviation(), 1E-14);
    assertEquals("Michelso: mean", mean, u.getMean(), 1E-14);   

    loadStats("data/NumAcc1.txt", u);
    assertEquals("NumAcc1: std", std, u.getStandardDeviation(), 1E-14);
    assertEquals("NumAcc1: mean", mean, u.getMean(), 1E-14);
    
    loadStats("data/NumAcc2.txt", u);
    assertEquals("NumAcc2: std", std, u.getStandardDeviation(), 1E-14);
    assertEquals("NumAcc2: mean", mean, u.getMean(), 1E-14);
}
 

/** {@inheritDoc} */
@Override
public long getN() {
    int n = 0;

    if (windowSize != DescriptiveStatistics.INFINITE_WINDOW) {
        if (list.size() > windowSize) {
            n = windowSize;
        } else {
            n = list.size();
        }
    } else {
        n = list.size();
    }
    return n;
}
 

/** {@inheritDoc} */
@Override
public double[] getValues() {

    int length = list.size();

    // If the window size is not INFINITE_WINDOW AND
    // the current list is larger that the window size, we need to
    // take into account only the last n elements of the list
    // as definied by windowSize

    if (windowSize != DescriptiveStatistics.INFINITE_WINDOW &&
        windowSize < list.size())
    {
        length = list.size() - FastMath.max(0, list.size() - windowSize);
    }

    // Create an array to hold all values
    double[] copiedArray = new double[length];

    for (int i = 0; i < copiedArray.length; i++) {
        copiedArray[i] = getElement(i);
    }
    return copiedArray;
}
 

/** {@inheritDoc} */
@Override
public double getElement(int index) {

    double value = Double.NaN;

    int calcIndex = index;

    if (windowSize != DescriptiveStatistics.INFINITE_WINDOW &&
        windowSize < list.size())
    {
        calcIndex = (list.size() - windowSize) + index;
    }


    try {
        value = transformer.transform(list.get(calcIndex));
    } catch (MathException e) {
        e.printStackTrace();
    }

    return value;
}