Java Code Examples for org.apache.commons.math3.stat.descriptive.SummaryStatistics#getMean()

/** test failure modes and distribution of nextGaussian() */
@Test
public void testNextGaussian() {
    try {
        randomData.nextGaussian(0, 0);
        Assert.fail("zero sigma -- MathIllegalArgumentException expected");
    } catch (MathIllegalArgumentException ex) {
        // ignored
    }
    SummaryStatistics u = new SummaryStatistics();
    for (int i = 0; i < largeSampleSize; i++) {
        u.addValue(randomData.nextGaussian(0, 1));
    }
    double xbar = u.getMean();
    double s = u.getStandardDeviation();
    double n = u.getN();
    /*
     * t-test at .001-level TODO: replace with externalized t-test, with
     * test statistic defined in TestStatistic
     */
    Assert.assertTrue(FastMath.abs(xbar) / (s / FastMath.sqrt(n)) < 3.29);
}
 

/**
 * Generates a random value from this distribution.
 * <strong>Preconditions:</strong><ul>
 * <li>the distribution must be loaded before invoking this method</li></ul>
 * @return the random value.
 * @throws MathIllegalStateException if the distribution has not been loaded
 */
public double getNextValue() throws MathIllegalStateException {

    if (!loaded) {
        throw new MathIllegalStateException(LocalizedFormats.DISTRIBUTION_NOT_LOADED);
    }

    // Start with a uniformly distributed random number in (0,1)
    double x = randomData.nextUniform(0,1);

    // Use this to select the bin and generate a Gaussian within the bin
    for (int i = 0; i < binCount; i++) {
       if (x <= upperBounds[i]) {
           SummaryStatistics stats = binStats.get(i);
           if (stats.getN() > 0) {
               if (stats.getStandardDeviation() > 0) {  // more than one obs
                    return randomData.nextGaussian
                        (stats.getMean(),stats.getStandardDeviation());
               } else {
                   return stats.getMean(); // only one obs in bin
               }
           }
       }
    }
    throw new MathIllegalStateException(LocalizedFormats.NO_BIN_SELECTED);
}
 

private double calcClusterMeanInfo(Set<Integer> members) throws AdeInternalException {
    if (members.size() == 1){
        return 0;
    }
    final SummaryStatistics sumS = new SummaryStatistics();
    for (int i : members){
        for (int j : members) {
            if (i <= j){
                continue;
            }
            double v;
            v = m_informationMat.get(i, j);
            if (!Double.isNaN(v)){
                sumS.addValue(v);
            }
        }
    }
    final double res = sumS.getMean();
    if (Double.isNaN(res)){
        return 0;
    }
    return res;

}
 

/** test failure modes and distribution of nextGaussian() */
@Test
public void testNextGaussian() {
    try {
        randomData.nextGaussian(0, 0);
        Assert.fail("zero sigma -- MathIllegalArgumentException expected");
    } catch (MathIllegalArgumentException ex) {
        // ignored
    }
    SummaryStatistics u = new SummaryStatistics();
    for (int i = 0; i < largeSampleSize; i++) {
        u.addValue(randomData.nextGaussian(0, 1));
    }
    double xbar = u.getMean();
    double s = u.getStandardDeviation();
    double n = u.getN();
    /*
     * t-test at .001-level TODO: replace with externalized t-test, with
     * test statistic defined in TestStatistic
     */
    Assert.assertTrue(FastMath.abs(xbar) / (s / FastMath.sqrt(n)) < 3.29);
}
 

private static Stats getStats(NumericColumn<?> values, SummaryStatistics summaryStatistics) {
  Stats stats = new Stats("Column: " + values.name());
  stats.min = summaryStatistics.getMin();
  stats.max = summaryStatistics.getMax();
  stats.n = summaryStatistics.getN();
  stats.sum = summaryStatistics.getSum();
  stats.variance = summaryStatistics.getVariance();
  stats.populationVariance = summaryStatistics.getPopulationVariance();
  stats.quadraticMean = summaryStatistics.getQuadraticMean();
  stats.geometricMean = summaryStatistics.getGeometricMean();
  stats.mean = summaryStatistics.getMean();
  stats.standardDeviation = summaryStatistics.getStandardDeviation();
  stats.sumOfLogs = summaryStatistics.getSumOfLogs();
  stats.sumOfSquares = summaryStatistics.getSumsq();
  stats.secondMoment = summaryStatistics.getSecondMoment();
  return stats;
}
 

/** test failure modes and distribution of nextGaussian() */
@Test
public void testNextGaussian() {
    try {
        randomData.nextGaussian(0, 0);
        Assert.fail("zero sigma -- MathIllegalArgumentException expected");
    } catch (MathIllegalArgumentException ex) {
        // ignored
    }
    SummaryStatistics u = new SummaryStatistics();
    for (int i = 0; i < largeSampleSize; i++) {
        u.addValue(randomData.nextGaussian(0, 1));
    }
    double xbar = u.getMean();
    double s = u.getStandardDeviation();
    double n = u.getN();
    /*
     * t-test at .001-level TODO: replace with externalized t-test, with
     * test statistic defined in TestStatistic
     */
    Assert.assertTrue(FastMath.abs(xbar) / (s / FastMath.sqrt(n)) < 3.29);
}
 

/**
 * Generates a random value from this distribution.
 * <strong>Preconditions:</strong><ul>
 * <li>the distribution must be loaded before invoking this method</li></ul>
 * @return the random value.
 * @throws MathIllegalStateException if the distribution has not been loaded
 */
public double getNextValue() throws MathIllegalStateException {

    if (!loaded) {
        throw new MathIllegalStateException(LocalizedFormats.DISTRIBUTION_NOT_LOADED);
    }

    // Start with a uniformly distributed random number in (0,1)
    double x = randomData.nextUniform(0,1);

    // Use this to select the bin and generate a Gaussian within the bin
    for (int i = 0; i < binCount; i++) {
       if (x <= upperBounds[i]) {
           SummaryStatistics stats = binStats.get(i);
           if (stats.getN() > 0) {
               if (stats.getStandardDeviation() > 0) {  // more than one obs
                    return randomData.nextGaussian
                        (stats.getMean(),stats.getStandardDeviation());
               } else {
                   return stats.getMean(); // only one obs in bin
               }
           }
       }
    }
    throw new MathIllegalStateException(LocalizedFormats.NO_BIN_SELECTED);
}
 

public double calculateSubgraphAnnotationSufficiencyForAttributeSet(Set<OWLClass> atts, OWLClass c) throws UnknownOWLClassException {
	SummaryStatistics stats = computeAttributeSetSimilarityStatsForSubgraph(atts,c);
	//TODO: compute statsPerIndividual for this subgraph
	if ((this.overallSummaryStatsPerIndividual == null ) || (Double.isNaN(this.overallSummaryStatsPerIndividual.max.getMean()))) {
		LOG.info("Stats have not been computed yet - doing this now");
		this.computeSystemStats();
	}

	if (!(this.subgraphSummaryStatsPerIndividual.containsKey(c))) {
		//only do this once for the whole system, per class requested
		this.computeSystemStatsForSubgraph(c);
	}
	// score = mean(atts)/mean(overall) + max(atts)/max(overall) + sum(atts)/mean(sum(overall))
	//TODO: need to normalize this based on the whole corpus
	double score = 0.0;
	Double mean_score = stats.getMean();
	Double max_score = stats.getMax();
	Double sum_score = stats.getSum();

	if (!(mean_score.isNaN() || max_score.isNaN() || sum_score.isNaN())) {
		mean_score = StatUtils.min(new double[]{(mean_score / this.subgraphSummaryStatsPerIndividual.get(c).mean.getMean()),1.0});
		max_score = StatUtils.min(new double[]{(max_score / this.subgraphSummaryStatsPerIndividual.get(c).max.getMax()),1.0});
		sum_score = StatUtils.min(new double[]{(sum_score / this.subgraphSummaryStatsPerIndividual.get(c).sum.getMean()),1.0});
		score = (mean_score + max_score + sum_score) / 3;		
	}
	LOG.info(getShortId(c)+" n: "+stats.getN()+" mean: "+mean_score + " max: "+max_score + " sum:"+sum_score + " combined:"+score);
	return score;
}
 

/**
 * The within-bin smoothing kernel.
 *
 * @param bStats summary statistics for the bin
 * @return within-bin kernel parameterized by bStats
 */
protected RealDistribution getKernel(SummaryStatistics bStats) {
    // Default to Gaussian
    return new NormalDistribution(randomData.getRandomGenerator(),
            bStats.getMean(), bStats.getStandardDeviation(),
            NormalDistribution.DEFAULT_INVERSE_ABSOLUTE_ACCURACY);
}
 

/**
 * Compute the mean of the image
 *
 * @param frame
 * @param colourSpace
 * @return
 */
public double[] computeMean(MBFImage frame, ColourSpace colourSpace) {
	final Circle hc = circle.clone();
	hc.scale(0.5f);
	final Rectangle bounds = hc.calculateRegularBoundingBox();

	frame = ResizeProcessor.halfSize(frame);

	final MBFImage cvt = colourSpace.convert(frame);
	final double[] vector = new double[colourSpace.getNumBands()];

	final SummaryStatistics stats = new SummaryStatistics();
	final Pixel pt = new Pixel();
	for (int b = 0; b < colourSpace.getNumBands(); b++) {
		stats.clear();

		final float[][] pix = cvt.getBand(b).pixels;

		for (pt.y = (int) bounds.y; pt.y < bounds.y + bounds.height; pt.y++) {
			for (pt.x = (int) bounds.x; pt.x < bounds.x + bounds.width; pt.x++) {
				if (hc.isInside(pt)) {
					stats.addValue(pix[pt.y][pt.x]);
				}
			}
		}

		vector[b] = stats.getMean();
	}
	return vector;
}
 

/**
 * The within-bin smoothing kernel.
 *
 * @param bStats summary statistics for the bin
 * @return within-bin kernel parameterized by bStats
 */
protected RealDistribution getKernel(SummaryStatistics bStats) {
    // Default to Gaussian
    return new NormalDistribution(randomData.getRandomGenerator(),
            bStats.getMean(), bStats.getStandardDeviation(),
            NormalDistribution.DEFAULT_INVERSE_ABSOLUTE_ACCURACY);
}
 

/**
 * The within-bin smoothing kernel. Returns a Gaussian distribution
 * parameterized by {@code bStats}, unless the bin contains only one
 * observation, in which case a constant distribution is returned.
 *
 * @param bStats summary statistics for the bin
 * @return within-bin kernel parameterized by bStats
 */
protected RealDistribution getKernel(SummaryStatistics bStats) {
    if (bStats.getN() == 1) {
        return new ConstantRealDistribution(bStats.getMean());
    } else {
        return new NormalDistribution(randomData.getRandomGenerator(),
            bStats.getMean(), bStats.getStandardDeviation(),
            NormalDistribution.DEFAULT_INVERSE_ABSOLUTE_ACCURACY);
    }
}
 

/**
 * Initializes the model using a training set of anomaly scores.
 *
 * The hybrid model initialization has several steps. First, a log-normal
 * distribution is fit to the training set scores. Next, the quantile sketch
 * is initialized with at {@code numLogNormalQuantiles} samples from the
 * log-normal model up to {@code maxScore}.
 *
 * @param anomalyScores  an array of anomaly scores with which to train the model.
 */
@Override
public void train(double[] anomalyScores) {
    /*
      We assume the anomaly scores are fit to a log-normal distribution.
      Equivalent to fitting a Gaussian to the logs of the anomaly scores.
    */
    SummaryStatistics stats = new SummaryStatistics();
    for (int i = 0; i < anomalyScores.length; i++) {
        stats.addValue(Math.log(anomalyScores[i]));
    }
    final double mu = stats.getMean();
    final double sigma = stats.getStandardDeviation();

    /*
      Compute the 1/R quantiles for R = `numLogNormalQuantiles` of the
      corresponding log-normal distribution and use these to initialize the
      model. We only compute p-values up to the p-value of the known maximum
      possible score. Finally, we do not compute the p=0.0 quantile because
      raw anomaly scores are positive and non-zero.
    */
    final double maxScorePvalue = computeLogNormalCdf(maxScore, mu, sigma);
    final double pvalueStep = maxScorePvalue / ((double) numLogNormalQuantiles + 1.0);
    for (double pvalue = pvalueStep; pvalue < maxScorePvalue; pvalue += pvalueStep) {
        double currentScore = computeLogNormalQuantile(pvalue, mu, sigma);
        update(currentScore);
    }
}
 

public static Long averageTimeBetweenEntries(List<FeedEntry> entries) {
	if (entries.isEmpty() || entries.size() == 1) {
		return null;
	}

	List<Long> timestamps = getSortedTimestamps(entries);

	SummaryStatistics stats = new SummaryStatistics();
	for (int i = 0; i < timestamps.size() - 1; i++) {
		long diff = Math.abs(timestamps.get(i) - timestamps.get(i + 1));
		stats.addValue(diff);
	}
	return (long) stats.getMean();
}
 

/**
 * The within-bin smoothing kernel.
 *
 * @param bStats summary statistics for the bin
 * @return within-bin kernel parameterized by bStats
 */
protected RealDistribution getKernel(SummaryStatistics bStats) {
    // Default to Gaussian
    return new NormalDistribution(randomData.getRandomGenerator(),
            bStats.getMean(), bStats.getStandardDeviation(),
            NormalDistribution.DEFAULT_INVERSE_ABSOLUTE_ACCURACY);
}
 

/**
 * Applies a global filter on pitches in S1 and moves all pitches whose salience is bellow a certain
 * threshold from S1 to S0. This filter is described in [1], section II-C.
 */
private void applyGlobalFilter() {
    SummaryStatistics statistics = new SummaryStatistics();

    /* Iteration #1: Gather data to obtain salience statistics. */
    for (int t=0; t<this.s1.length; t++) {
        for (int i=0; i<this.s1[t].length; i++) {
            if (this.s1[t][i] == null) {
              continue;
            }
            statistics.addValue(this.s1[t][i].getSalience());
        }
    }

    /* Iteration #2: Move pitches that are bellow the threshold. */
    final double threshold = statistics.getMean() - this.t2 * statistics.getStandardDeviation();
    for (int t=0; t<this.s1.length; t++) {
        for (int i=0; i<this.s1[t].length; i++) {
            if (this.s1[t][i] == null) {
              continue;
            }
            if (this.s1[t][i].getSalience() < threshold) {
                this.moveToS0(t,i);
            }
        }
    }
}
 

/**
 * The within-bin smoothing kernel. Returns a Gaussian distribution
 * parameterized by {@code bStats}, unless the bin contains only one
 * observation, in which case a constant distribution is returned.
 *
 * @param bStats summary statistics for the bin
 * @return within-bin kernel parameterized by bStats
 */
protected RealDistribution getKernel(SummaryStatistics bStats) {
    if (bStats.getN() == 1) {
        return new ConstantRealDistribution(bStats.getMean());
    } else {
        return new NormalDistribution(randomData.getRandomGenerator(),
            bStats.getMean(), bStats.getStandardDeviation(),
            NormalDistribution.DEFAULT_INVERSE_ABSOLUTE_ACCURACY);
    }
}
 

@Override
protected RealDistribution getKernel(SummaryStatistics bStats) {
    return new ConstantRealDistribution(bStats.getMean());
}
 

protected RealDistribution getKernel(SummaryStatistics bStats) {
    return new ConstantDistribution(bStats.getMean());
}
 

@VisibleForTesting
BigDecimal calculateDeviation(Context context, Request request) {

    Instant to = request.getCurrentTime();

    Duration interval = Duration.between(to, request.getTargetTime());

    Instant from = request.getCurrentTime().minus(interval.toMillis() * getSamples(), MILLIS);

    List<Trade> trades = context.listTrades(getKey(context, request), from.minus(interval));

    NavigableMap<Instant, BigDecimal> prices = collapsePrices(trades, interval, from, to, false);

    NavigableMap<Instant, BigDecimal> returns = calculateReturns(prices);

    SummaryStatistics stats = new SummaryStatistics();

    returns.values().stream().filter(Objects::nonNull).forEach(r -> stats.addValue(r.doubleValue()));

    if (stats.getN() <= 1) {
        return null;
    }

    double avg = stats.getMean();

    double dev = stats.getStandardDeviation();

    double sigma = new TDistribution(stats.getN() - 1).inverseCumulativeProbability(PROBABILITY);

    double sum = Math.abs(avg) + (dev * sigma);

    return Double.isFinite(sum) ? BigDecimal.valueOf(sum).setScale(SCALE, HALF_UP) : null;

}