Java Code Examples for org.apache.commons.math3.stat.regression.OLSMultipleLinearRegression#estimateRegressionParameters()

/**
 * Implements ordinary least squares (OLS) to estimate the parameters of a 
 * multiple linear regression model.
 * @param y Y sample data - one dimension array
 * @param x X sample data - two dimension array
 * @param noIntercept No intercept
 * @return Estimated regression parameters and residuals
 */
public static Array[] multipleLineRegress_OLS(Array y, Array x, boolean noIntercept) {
    OLSMultipleLinearRegression regression = new OLSMultipleLinearRegression();
    regression.setNoIntercept(noIntercept);
    double[] yy = (double[])ArrayUtil.copyToNDJavaArray_Double(y);
    double[][] xx = (double[][])ArrayUtil.copyToNDJavaArray_Double(x);
    regression.newSampleData(yy, xx);
    double[] para = regression.estimateRegressionParameters();
    double[] residuals = regression.estimateResiduals();
    int k = para.length;
    int n = residuals.length;
    Array aPara = Array.factory(DataType.DOUBLE, new int[]{k});
    Array aResiduals = Array.factory(DataType.DOUBLE, new int[]{n});
    for (int i = 0; i < k; i++){
        aPara.setDouble(i, para[i]);
    }
    for (int i = 0; i < k; i++){
        aResiduals.setDouble(i, residuals[i]);
    }
    
    return new Array[]{aPara, aResiduals};
}
 

/**
 * Trigger the calculation of the model parameters.
 * @return True if the parameters are generated, otherwise false;
 */
public synchronized boolean updateModelCoefficient() {
  if (validBuckets().size() < MIN_CPU_UTIL_OBSERVATION_BUCKETS) {
    return false;
  }
  try {
    OLSMultipleLinearRegression regression = new OLSMultipleLinearRegression();
    regression.setNoIntercept(true);
    boolean ignoreLeaderBytesOut = !isLeaderBytesInAndOutRatioDiverseEnough();
    regression.newSampleData(aggregateSampleCpuUtilData(),
                             aggregateSampleBytesRateData(ignoreLeaderBytesOut));
    double[] parameters = regression.estimateRegressionParameters();
    int leaderBytesInIndex = 0;
    int leaderBytesOutIndex = 1;
    int followerBytesInIndex = ignoreLeaderBytesOut ? 1 : 2;
    _coefficients.put(ModelCoefficient.LEADER_BYTES_IN, parameters[leaderBytesInIndex]);
    if (!ignoreLeaderBytesOut) {
      _coefficients.put(ModelCoefficient.LEADER_BYTES_OUT, parameters[leaderBytesOutIndex]);
    }
    _coefficients.put(ModelCoefficient.FOLLOWER_BYTES_IN, parameters[followerBytesInIndex]);

    LOG.info("Coefficient generated: leader_bytes_in: {}, leader_bytes_out: {}, follower_bytes_in: {}",
             _coefficients.get(ModelCoefficient.LEADER_BYTES_IN),
             _coefficients.get(ModelCoefficient.LEADER_BYTES_OUT),
             _coefficients.get(ModelCoefficient.FOLLOWER_BYTES_IN));
    return true;
  } catch (Exception e) {
    LOG.warn("received exception {}", e);
  }
  return false;
}
 

public void calculateOlsRegression(double[][] x, double[] y){
	OLSMultipleLinearRegression regression = new OLSMultipleLinearRegression();
	regression.newSampleData(y, x);
	
	double[] beta = regression.estimateRegressionParameters();       
	double[] residuals = regression.estimateResiduals();
	double[][] parametersVariance = regression.estimateRegressionParametersVariance();
	double regressandVariance = regression.estimateRegressandVariance();
	double rSquared = regression.calculateRSquared();
	double sigma = regression.estimateRegressionStandardError();
}
 

/**
 * Checks that the Morpheus OLS model yields the same results as Apache Math
 * @param actual    the Morpheus results
 * @param expected  the Apache results
 */
private <R,C> void assertResultsMatch(DataFrameLeastSquares<R,C> actual, OLSMultipleLinearRegression expected) {

    Assert.assertEquals(actual.getResidualSumOfSquares(), expected.calculateResidualSumOfSquares(), 0.0000001, "Residual sum of squares matches");
    Assert.assertEquals(actual.getTotalSumOfSquares(), expected.calculateTotalSumOfSquares(), actual.getTotalSumOfSquares() * 0.000001, "Total sum of squares matches");
    Assert.assertEquals(actual.getRSquared(), expected.calculateRSquared(), 0.0000001, "R^2 values match");
    Assert.assertEquals(actual.getStdError(), expected.estimateRegressionStandardError(),  0.0000001, "Std error matches");

    final DataFrame<C,Field> params1 = actual.getBetas();
    final double[] params2 = expected.estimateRegressionParameters();
    Assert.assertEquals(params1.rows().count(), params2.length-1, "Same number of parameters");
    for (int i=0; i<params1.rows().count(); ++i) {
        final double actualParam = params1.data().getDouble(i, Field.PARAMETER);
        final double expectedParam = params2[i+1];
        Assert.assertEquals(actualParam, expectedParam, 0.000000001, "Parameters match at index " + i);
    }

    final double intercept = expected.estimateRegressionParameters()[0];
    final double interceptStdError = expected.estimateRegressionParametersStandardErrors()[0];
    Assert.assertEquals(actual.getInterceptValue(Field.PARAMETER), intercept,  0.0000001, "The intercepts match");
    Assert.assertEquals(actual.getInterceptValue(Field.STD_ERROR), interceptStdError, 0.000000001, "The intercept std errors match");

    final DataFrame<R,String> residuals1 = actual.getResiduals();
    final double[] residuals2 = expected.estimateResiduals();
    Assert.assertEquals(residuals1.rows().count(), residuals2.length, "Same number of residuals");
    for (int i=0; i<residuals1.rows().count(); ++i) {
        Assert.assertEquals(residuals1.data().getDouble(i, 0), residuals2[i], 0.00000001, "Residuals match at index " + i);
    }

    final DataFrame<C,Field> stdErrs1 = actual.getBetas().cols().select(c -> c.key() == Field.STD_ERROR);
    final double[] stdErrs2 = expected.estimateRegressionParametersStandardErrors();
    Assert.assertEquals(stdErrs1.rows().count(), stdErrs2.length-1, "Same number of parameter standard errors");
    for (int i=0; i<stdErrs1.cols().count(); ++i) {
        Assert.assertEquals(stdErrs1.data().getDouble(0, i), stdErrs2[i+1], 0.00000001, "Standard errors match at index " + i);
    }
}
 

/**
 * @return Linear regression model state.
 */
public synchronized LinearRegressionModelState modelState() {
  Map<Integer, Double> detailCompleteness = new HashMap<>();
  for (Map.Entry<Integer, AtomicInteger> entry : INDICES.entrySet()) {
    detailCompleteness.put(entry.getKey(),
                           Math.min((double) entry.getValue().get() / NUM_OBSERVATIONS_PER_UTIL_BUCKET, 1.0));
  }
  Map<Integer, Integer> usedLeaderToFollowerRatio = new HashMap<>();
  Map<Integer, Integer> usedLeaderBytesInToBytesOutRatio = new HashMap<>();
  Map<ModelCoefficient, Double> coefficientFromAvailableData = new HashMap<>(_coefficients);
  OLSMultipleLinearRegression regression = new OLSMultipleLinearRegression();
  regression.setNoIntercept(true);
  boolean ignoreLeaderBytesOutRate = !isLeaderBytesInAndOutRatioDiverseEnough();
  double[][] sampleBytesRateData = aggregateSampleBytesRateData(ignoreLeaderBytesOutRate);

  int leaderBytesInIndex = 0;
  int leaderBytesOutIndex = 1;
  int followerBytesInIndex = ignoreLeaderBytesOutRate ? 1 : 2;
  for (int i = 0; i < sampleBytesRateData.length; i++) {
    int leaderToFollowerRatio = sampleBytesRateData[i][followerBytesInIndex] == 0.0 ? 10000000 :
        (int) ((sampleBytesRateData[i][leaderBytesInIndex] / sampleBytesRateData[i][followerBytesInIndex]) * 10);
    int count = usedLeaderToFollowerRatio.getOrDefault(leaderToFollowerRatio, 0);
    usedLeaderToFollowerRatio.put(leaderToFollowerRatio, count + 1);

    if (!ignoreLeaderBytesOutRate) {
      int leaderBytesInToBytesOutRatio = sampleBytesRateData[i][leaderBytesOutIndex] == 0.0 ? 10000000 :
          (int) ((sampleBytesRateData[i][leaderBytesInIndex] / sampleBytesRateData[i][leaderBytesOutIndex]) * 10);
      count = usedLeaderBytesInToBytesOutRatio.getOrDefault(leaderBytesInToBytesOutRatio, 0);
      usedLeaderBytesInToBytesOutRatio.put(leaderBytesInToBytesOutRatio, count + 1);
    }
  }
  regression.newSampleData(aggregateSampleCpuUtilData(), sampleBytesRateData);
  double[] parameters = regression.estimateRegressionParameters();
  coefficientFromAvailableData.put(ModelCoefficient.LEADER_BYTES_IN, parameters[leaderBytesInIndex]);
  if (ignoreLeaderBytesOutRate) {
    coefficientFromAvailableData.put(ModelCoefficient.FOLLOWER_BYTES_IN, parameters[followerBytesInIndex]);
  } else {
    coefficientFromAvailableData.put(ModelCoefficient.LEADER_BYTES_OUT, parameters[leaderBytesOutIndex]);
    coefficientFromAvailableData.put(ModelCoefficient.FOLLOWER_BYTES_IN, parameters[followerBytesInIndex]);
  }
  return new LinearRegressionModelState(detailCompleteness, coefficientFromAvailableData,
                                        OBSERVED_LEADER_TO_FOLLOWER_BYTES_RATIO,
                                        OBSERVED_LEADER_BYTES_IN_TO_BYTES_OUT_RATIO,
                                        usedLeaderToFollowerRatio, usedLeaderBytesInToBytesOutRatio,
                                        CPU_UTIL_ESTIMATION_ERROR_STATS);
}
 

public static void main(String[] args) {
    OLSMultipleLinearRegression mlr = new OLSMultipleLinearRegression();
    mlr.newSampleData(y, x);
    double[] b = mlr.estimateRegressionParameters();
    printResults(b);
}