Java Code Examples for net.sourceforge.openforecast.DataSet#iterator()

/**
 * Writes a DataSet - a collection of DataPoints - to the current writer.
 * The DataSet should contain all DataPoints to be output.
 *
 * <p>Depending on the setting of outputHeaderRow, a header row containing
 * the variable names of the data points will be output. To enable/disable
 * this feature, use the {@link #setOutputHeaderRow} method.</li>
 * @param dataSet the DataSet to be output to the current writer.
 * @throws IOException if an I/O error occurs.
 */
public void output( DataSet dataSet )
    throws IOException
{
    if ( outputHeaderRow )
        writeHeader( dataSet.getIndependentVariables() );
    
    Iterator<DataPoint> it = dataSet.iterator();
    while ( it.hasNext() )
        {
            DataPoint dataPoint = it.next();
            output( dataPoint );
        }
    
    out.flush();
}
 

/**
 * Tests the correct initialization of a DataSet.
 */
public void testDataSet()
{
    DataSet data = new DataSet( dataSet1 );
    
    // Verify data set contains the correct number of entries
    assertTrue( data.size() == dataSet1.size() );
    
    // Vefify that only one independent variable name is reported
    String[] independentVariables = data.getIndependentVariables();
    assertTrue( independentVariables.length == 1 );
    assertTrue( independentVariables[0].equals("x") );
    
    // Verify the dependent values stored
    Iterator<DataPoint> it = data.iterator();
    while ( it.hasNext() )
        {
            DataPoint dp = it.next();
            double value = dp.getDependentValue();
            double TOLERANCE = 0.001;
            assertTrue( value>-TOLERANCE && value<SIZE+TOLERANCE );
        }
}
 

private double getLastValue(DataSet forecasted) {
    Iterator<DataPoint> itr = forecasted.iterator();
    while (itr.hasNext()) {
        DataPoint dp = itr.next();
        if (! itr.hasNext()) {
            return dp.getDependentValue();
        }
    }
    return 0;
}
 

/**
 * Use the given forecasting model to produce a TimeSeries object
 * representing the periods startIndex through endIndex, and containing
 * the forecast values produced by the model.
 * @param model the forecasting model to use to generate the forecast
 * series.
 * @param initDataSet data set to use to initialize the forecasting model.
 * @param startIndex the index of the first data point to use from the
 * set of potential forecast values.
 * @param endIndex the index of the last data point to use from the set
 * of potential forecast values.
 * @param title a title to give to the TimeSeries created.
 */
private TimeSeries getForecastTimeSeries( ForecastingModel model,
                                               DataSet initDataSet,
                                               int startIndex,
                                               int endIndex,
                                               String title )
{
    // Initialize the forecasting model
    model.init( initDataSet );
    
    // Get range of data required for forecast
    DataSet fcDataSet = getDataSet( fc, startIndex, endIndex );
    
    // Obtain forecast values for the forecast data set
    model.forecast( fcDataSet );

    // Create a new TimeSeries
    TimeSeries series
        = new TimeSeries(title,fc.getTimePeriodClass());
    
    // Iterator through the forecast results, adding to the series
    Iterator it = fcDataSet.iterator();
    while ( it.hasNext() )
        {
            DataPoint dp = (DataPoint)it.next();
            int index = (int)dp.getIndependentValue("t");
            series.add( fc.getTimePeriod(index), dp.getDependentValue() );
        }
    
    return series;
}
 

/**
 * Use the given forecasting model to produce a TimeSeries object
 * representing the periods startIndex through endIndex, and containing
 * the forecast values produced by the model.
 * @param model the forecasting model to use to generate the forecast
 * series.
 * @param initDataSet data set to use to initialize the forecasting model.
 * @param startIndex the index of the first data point to use from the
 * set of potential forecast values.
 * @param endIndex the index of the last data point to use from the set
 * of potential forecast values.
 * @param title a title to give to the TimeSeries created.
 */
private TimeSeries getForecastTimeSeries( ForecastingModel model,
                                          DataSet initDataSet,
                                          int startIndex,
                                          int endIndex,
                                          String title )
{
    // Initialize the forecasting model
    model.init( initDataSet );
    
    // Get range of data required for forecast
    DataSet fcDataSet = getDataSet( fc, startIndex, endIndex );
    
    // Obtain forecast values for the forecast data set
    model.forecast( fcDataSet );
    
    // Create a new TimeSeries
    TimeSeries series
        = new TimeSeries(title,fc.getTimePeriodClass());
    
    // Iterator through the forecast results, adding to the series
    Iterator it = fcDataSet.iterator();
    while ( it.hasNext() )
        {
            DataPoint dp = (DataPoint)it.next();
            int index = (int)dp.getIndependentValue("t");
            series.add( fc.getTimePeriod(index), dp.getDependentValue() );
        }
    
    return series;
}
 

/**
 * Used to initialize the time based model. This method must be called
 * before any other method in the class. Since the time based model does
 * not derive any equation for forecasting, this method uses the input
 * DataSet to calculate forecast values for all values of the independent
 * time variable within the initial data set.
 * @param dataSet a data set of observations that can be used to
 * initialize the forecasting parameters of the forecasting model.
 */
public void init( DataSet dataSet )
{
    initTimeVariable( dataSet );
    String timeVariable = getTimeVariable();

    if ( dataSet.getPeriodsPerYear() <= 1 )
        throw new IllegalArgumentException("Data set passed to init in the triple exponential smoothing model does not contain seasonal data. Don't forget to call setPeriodsPerYear on the data set to set this.");

    periodsPerYear = dataSet.getPeriodsPerYear();

    // Check we have the minimum amount of data points
    if ( dataSet.size() < NUMBER_OF_YEARS*periodsPerYear )
        throw new IllegalArgumentException("TripleExponentialSmoothing models require a minimum of a full two years of data to initialize the model.");

    // Calculate initial values for base and trend
    initBaseAndTrendValues( dataSet );

    // Initialize seasonal indices using data for all complete years
    initSeasonalIndices( dataSet );

    Iterator<DataPoint> it = dataSet.iterator();
    maxObservedTime = Double.NEGATIVE_INFINITY;
    while ( it.hasNext() )
        {
            DataPoint dp = it.next();
            if ( dp.getIndependentValue(timeVariable) > maxObservedTime )
                maxObservedTime = dp.getIndependentValue(timeVariable);
        }

    super.init( dataSet );
}
 

/**
 * Initializes the coefficients to use for this regression model. The
 * intercept and slope are derived so as to give the best fit line for the
 * given data set.
 *
 * <p>Additionally, the accuracy indicators are calculated based on this
 * data set.
 * @param dataSet the set of observations to use to derive the regression
 * coefficients for this model.
 */
public void init( DataSet dataSet )
{
    int n = dataSet.size();
    double sumX = 0.0;
    double sumY = 0.0;
    double sumXX = 0.0;
    double sumXY = 0.0;
    
    Iterator<DataPoint> it = dataSet.iterator();
    while ( it.hasNext() )
        {
            DataPoint dp = it.next();
            
            double x = dp.getIndependentValue( independentVariable );
            double y = dp.getDependentValue();
            
            sumX += x;
            sumY += y;
            sumXX += x*x;
            sumXY += x*y;
        }
    
    double xMean = sumX / n;
    double yMean = sumY / n;
    
    slope = (n*sumXY - sumX*sumY) / (n*sumXX - sumX*sumX);
    intercept = yMean - slope*xMean;
    
    // Calculate the accuracy of this model
    calculateAccuracyIndicators( dataSet );
}
 

/**
 * Adds a DataSet - a collection of DataPoints - to the current TimeSeries.
 * The DataSet should contain all DataPoints to be output.
 * @param dataSet the DataSet to be output to the current TimeSeries.
 */
public void output( DataSet dataSet )
    throws InstantiationException, IllegalAccessException,
    InvocationTargetException, InstantiationException
{
    String timeVariable = dataSet.getTimeVariable();
    
    Iterator<DataPoint> it = dataSet.iterator();
    while ( it.hasNext() )
        {
            DataPoint dataPoint = it.next();
            output( dataPoint, timeVariable );
        }
}
 

/**
 * A helper function that validates the actual results obtaining for
 * a DataSet match the expected results. This is the same as the other
 * checkResults method except that with this method, the caller can
 * specify an acceptance tolerance when comparing actual with expected
 * results.
 * @param actualResults the DataSet returned from the forecast method
 *        that contains the data points for which forecasts were done.
 * @param expectedResults an array of expected values for the forecast
 *        data points. The order should match the order of the results
 *        as defined by the DataSet iterator.
 * @param tolerance the tolerance to accept when comparing the actual
 *        results (obtained from a forecasting model) with the expected
 *        results.
 */
protected void checkResults( DataSet actualResults,
                             double[] expectedResults,
                             double tolerance )
{
    // This is just to safeguard against a bug in the test case!  :-)
    assertNotNull( "Checking expected results is not null",
                   expectedResults );
    assertTrue( "Checking there are some expected results",
                expectedResults.length > 0 );
    
    assertEquals( "Checking the correct number of results returned",
                  expectedResults.length, actualResults.size() );
    
    // Iterate through the results, checking each one in turn
    Iterator<DataPoint> it = actualResults.iterator();
    int i=0;
    while ( it.hasNext() )
        {
            // Check that the results are within specified tolerance
            //  of the expected values
            DataPoint fc = (DataPoint)it.next();
            double fcValue = fc.getDependentValue();
            
            assertEquals( "Checking result",
                          expectedResults[i], fcValue, tolerance );
            i++;
        }
}
 

/**
 * Since this version of triple exponential smoothing uses the current
 * observation to calculate a smoothed value, we must override the
 * calculation of the accuracy indicators.
 * @param dataSet the DataSet to use to evaluate this model, and to
 *        calculate the accuracy indicators against.
 */
protected void calculateAccuracyIndicators( DataSet dataSet )
{
    // WARNING: THIS STILL NEEDS TO BE VALIDATED

    // Note that the model has been initialized
    initialized = true;

    // Reset various helper summations
    double sumErr = 0.0;
    double sumAbsErr = 0.0;
    double sumAbsPercentErr = 0.0;
    double sumErrSquared = 0.0;

    String timeVariable = getTimeVariable();
    double timeDiff = getTimeInterval();

    // Calculate the Sum of the Absolute Errors
    Iterator<DataPoint> it = dataSet.iterator();
    while ( it.hasNext() )
        {
            // Get next data point
            DataPoint dp = it.next();
            double x = dp.getDependentValue();
            double time = dp.getIndependentValue( timeVariable );
            double previousTime = time - timeDiff;

            // Get next forecast value, using one-period-ahead forecast
            double forecastValue
                = getForecastValue( previousTime )
                + getTrend( previousTime );

            // Calculate error in forecast, and update sums appropriately
            double error = forecastValue - x;
            sumErr += error;
            sumAbsErr += Math.abs( error );
            sumAbsPercentErr += Math.abs( error / x );
            sumErrSquared += error*error;
        }

    // Initialize the accuracy indicators
    int n = dataSet.size();

    accuracyIndicators.setBias( sumErr / n );
    accuracyIndicators.setMAD( sumAbsErr / n );
    accuracyIndicators.setMAPE( sumAbsPercentErr / n );
    accuracyIndicators.setMSE( sumErrSquared / n );
    accuracyIndicators.setSAE( sumAbsErr );
}
 

/**
 * Since this version of double exponential smoothing uses the current
 * observation to calculate a smoothed value, we must override the
 * calculation of the accuracy indicators.
 * @param dataSet the DataSet to use to evaluate this model, and to
 *        calculate the accuracy indicators against.
 */
protected void calculateAccuracyIndicators( DataSet dataSet )
{
    // Note that the model has been initialized
    initialized = true;
    
    // Reset various helper summations
    double sumErr = 0.0;
    double sumAbsErr = 0.0;
    double sumAbsPercentErr = 0.0;
    double sumErrSquared = 0.0;
    
    String timeVariable = getTimeVariable();
    double timeDiff = getTimeInterval();
    
    // Calculate the Sum of the Absolute Errors
    Iterator<DataPoint> it = dataSet.iterator();
    while ( it.hasNext() )
        {
            // Get next data point
            DataPoint dp = it.next();
            double x = dp.getDependentValue();
            double time = dp.getIndependentValue( timeVariable );
            double previousTime = time - timeDiff;
            
            // Get next forecast value, using one-period-ahead forecast
            double forecastValue
                = getForecastValue( previousTime )
                + getSlope( previousTime );
            
            // Calculate error in forecast, and update sums appropriately
            double error = forecastValue - x;
            sumErr += error;
            sumAbsErr += Math.abs( error );
            sumAbsPercentErr += Math.abs( error / x );
            sumErrSquared += error*error;
        }
    
    // Initialize the accuracy indicators
    int n = dataSet.size();
    
    accuracyIndicators.setBias( sumErr / n );
    accuracyIndicators.setMAD( sumAbsErr / n );
    accuracyIndicators.setMAPE( sumAbsPercentErr / n );
    accuracyIndicators.setMSE( sumErrSquared / n );
    accuracyIndicators.setSAE( sumAbsErr );
}
 

/**
 * Used to initialize the time based model. This method must be called
 * before any other method in the class. Since the time based model does
 * not derive any equation for forecasting, this method uses the input
 * DataSet to calculate forecast values for all values of the independent
 * time variable within the initial data set.
 * @param dataSet a data set of observations that can be used to initialize
 *        the forecasting parameters of the forecasting model.
 */
public void init( DataSet dataSet )
{
    initTimeVariable( dataSet );
    
    if ( dataSet == null  || dataSet.size() == 0 )
        throw new IllegalArgumentException("Data set cannot be empty in call to init.");
    
    int minPeriods = getNumberOfPeriods();
    
    if ( dataSet.size() < minPeriods )
        throw new IllegalArgumentException("Data set too small. Need "
                                           +minPeriods
                                           +" data points, but only "
                                           +dataSet.size()
                                           +" passed to init.");
    
    observedValues = new DataSet( dataSet );
    observedValues.sort( timeVariable );
    
    // Check that intervals between data points are consistent
    //  i.e. check for complete data set
    Iterator<DataPoint> it = observedValues.iterator();
    
    DataPoint dp = it.next();  // first data point
    double lastValue = dp.getIndependentValue(timeVariable);
    
    dp = it.next();  // second data point
    double currentValue = dp.getIndependentValue(timeVariable);
    
    // Create data set in which to save new forecast values
    forecastValues = new DataSet();
    
    // Determine "standard"/expected time difference between observations
    timeDiff = currentValue - lastValue;
    
    // Min. time value is first observation time
    minTimeValue = lastValue;
    
    while ( it.hasNext() )
        {
            lastValue = currentValue;
            
            // Get next data point
            dp = it.next();
            currentValue = dp.getIndependentValue(timeVariable);
            
            double diff = currentValue - lastValue;
            if ( Math.abs(timeDiff - diff) > TOLERANCE )
                throw new IllegalArgumentException( "Inconsistent intervals found in time series, using variable '"+timeVariable+"'" );
            
            try
                {
                    initForecastValue( currentValue );
                }
            catch (IllegalArgumentException ex)
                {
                    // We can ignore these during initialization
                }
        }
    
    // Create test data set for determining accuracy indicators
    //  - same as input data set, but without the first n data points
    DataSet testDataSet = new DataSet( observedValues );
    int count = 0;
    while ( count++ < minPeriods )
        testDataSet.remove( (testDataSet.iterator()).next() );
    
    // Calculate accuracy
    calculateAccuracyIndicators( testDataSet );
}
 

/**
 * Initializes the coefficients to use for this regression model. The
 * coefficients are derived so as to give the best fit hyperplane for the
 * given data set.
 *
 * <p>Additionally, the accuracy indicators are calculated based on this
 * data set.
 * @param dataSet the set of observations to use to derive the regression
 *        coefficients for this model.
 */
public void init( DataSet dataSet )
{
    String varNames[] = dataSet.getIndependentVariables();
    
    // If no coefficients have been defined for this model,
    //  use all that exist in this data set
    if ( coefficient == null )
        setIndependentVariables( varNames );
    
    int n = varNames.length;
    double a[][] = new double[n+1][n+2];
    
    // Iterate through dataSet
    Iterator<DataPoint> it = dataSet.iterator();
    while ( it.hasNext() )
        {
            // Get next data point
            DataPoint dp = it.next();
            
            // For each row in the matrix, a
            for ( int row=0; row<n+1; row++ )
                {
                    double rowMult = 1.0;
                    if ( row != 0 )
                        {
                            // Get value of independent variable, row
                            String rowVarName = varNames[row-1];
                            rowMult = dp.getIndependentValue(rowVarName);
                        }
                    
                    // For each column in the matrix, a
                    for ( int col=0; col<n+2; col++ )
                        {
                            double colMult = 1.0;
                            if ( col == n+1 )
                                {
                                    // Special case, for last column
                                    //  use value of dependent variable
                                    colMult = dp.getDependentValue();
                                }
                            else if ( col > 0 )
                                {
                                    // Get value of independent variable, col
                                    String colVarName = varNames[col-1];
                                    colMult = dp.getIndependentValue(colVarName);
                                }
                            
                            a[row][col] += rowMult * colMult;
                        }
                }
        }
    
    // Solve equations to derive coefficients
    double coeff[] = Utils.GaussElimination( a );
    
    // Assign coefficients to independent variables
    intercept = coeff[0];
    for ( int i=1; i<n+1; i++ )
        coefficient.put( varNames[i-1], new Double(coeff[i]) );
    
    // Calculate the accuracy indicators
    calculateAccuracyIndicators( dataSet );
}
 

/**
 * A helper method to calculate the various accuracy indicators when
 * applying the given DataSet to the current forecasting model.
 * @param dataSet the DataSet to use to evaluate this model, and to
 *        calculate the accuracy indicators against.
 */
protected void calculateAccuracyIndicators( DataSet dataSet )
{
    // Note that the model has been initialized
    initialized = true;
    
    // Reset various helper summations
    double sumErr = 0.0;
    double sumAbsErr = 0.0;
    double sumAbsPercentErr = 0.0;
    double sumErrSquared = 0.0;
    
    // Obtain the forecast values for this model
    DataSet forecastValues = new DataSet( dataSet );
    forecast( forecastValues );
    
    // Calculate the Sum of the Absolute Errors
    Iterator<DataPoint> it = dataSet.iterator();
    Iterator<DataPoint> itForecast = forecastValues.iterator();
    while ( it.hasNext() )
        {
            // Get next data point
            DataPoint dp = it.next();
            double x = dp.getDependentValue();
            
            // Get next forecast value
            DataPoint dpForecast = itForecast.next();
            double forecastValue = dpForecast.getDependentValue();
            
            // Calculate error in forecast, and update sums appropriately
            double error = forecastValue - x;
            sumErr += error;
            sumAbsErr += Math.abs( error );
            sumAbsPercentErr += Math.abs( error / x );
            sumErrSquared += error*error;
        }
    
    // Initialize the accuracy indicators
    int n = dataSet.size();
    int p = getNumberOfPredictors();

    accuracyIndicators.setAIC( n*Math.log(2*Math.PI)
               + Math.log(sumErrSquared/n)
               + 2 * ( p+2 ) );
    accuracyIndicators.setBias( sumErr / n );
    accuracyIndicators.setMAD( sumAbsErr / n );
    accuracyIndicators.setMAPE( sumAbsPercentErr / n );
    accuracyIndicators.setMSE( sumErrSquared / n );
    accuracyIndicators.setSAE( sumAbsErr );
}
 

/**
 * Tests the correct initialization of a DataSet from a CSV file where
 * the input is valid, yet poorly and irregularly formatted. For example,
 * the CSVBuilder is supposed to treat as a zero field two commas following
 * each other. This test will also test naming the columns and the use of
 * blank lines and comments in the input.
 */
public void testExtremeCSVBuilder()
    throws FileNotFoundException, IOException
{
    // Constants used to determine size of test
    double expectedValue[] = { 4,5,6,7,8 };
    int numberOfDataPoints = expectedValue.length;
    
    // Create test CSV file
    File testFile = File.createTempFile( "test", ".csv" );
    PrintStream out = new PrintStream( new FileOutputStream(testFile) );
    out.println("# This is a test CSV file with various 'peculiarities'");
    out.println(" # thrown in to try and trip it up");
    out.println("Field1, Field2, \"Field, 3\", Observation");
    out.println("-1, -2 ,-3,4");
    out.println(",,,5");
    out.println(" 1 , 2 , 3 , 6 ");
    out.println(" 2, 4, 6, 7");
    out.println("3 ,6 ,9 ,8");
    out.close();
    
    // Create CSV builder and use it to create the DataSet
    CSVBuilder builder = new CSVBuilder( testFile, true );
    DataSet dataSet = builder.build();
    
    // Verify data set contains the correct number of entries
    assertEquals( "DataSet created is of the wrong size",
                  numberOfDataPoints, dataSet.size() );
    
    // Vefify that only three independent variable names are reported
    String[] independentVariables = dataSet.getIndependentVariables();
    assertEquals( "Checking the correct number of independent variables",
                  3, independentVariables.length );
    
    // Note these will have been sorted into alphabetical order
    assertTrue( "Checking variable 0 name is as expected",
                independentVariables[0].compareTo("Field, 3")==0 );
    assertTrue( "Checking variable 1 name is as expected",
                independentVariables[1].compareTo("Field1")==0 );
    assertTrue( "Checking variable 2 name is as expected",
                independentVariables[2].compareTo("Field2")==0 );
    
    // Test the data set created by the builder
    Iterator<DataPoint> it = dataSet.iterator();
    while ( it.hasNext() )
        {
            DataPoint dataPoint = it.next();
            double field1 = dataPoint.getIndependentValue("Field1");
            double field2 = dataPoint.getIndependentValue("Field2");
            double field3 = dataPoint.getIndependentValue("Field, 3");
            
            // field2 was set to twice field1
            // field3 was set to three times field1
            assertTrue( "Checking independent values are correct",
                        field2==2*field1 && field3==3*field1 );
            
            // The data was set up with this simple equation
            double expectedResult = 5.0 + field1;
            
            assertEquals("Checking data point "+dataPoint,
                         expectedResult, dataPoint.getDependentValue(),
                         TOLERANCE);
        }
    
    // Clean up - remove test file
    testFile.delete();
}
 

/**
 * Using the current model parameters (initialized in init), apply the
 * forecast model to the given data set. Each data point in the data set
 * must have valid values for the independent variables. Upon return, the
 * value of the dependent variable will be updated with the forecast
 * values computed.
 *
 * This method is provided as a convenience method, and iterates through the
 * data set invoking forecast(DataPoint) to do the actual forecast for each
 * data point. In general, it is not necessary to override this method.
 * However, if a subclass can provide a more efficient approach then it is
 * recommended that the subclass provide its own implementation.
 * @param dataSet the set of data points for which forecast values (for
 *        the dependent variable) are required.
 * @return the same data set passed in but with the dependent values
 *         updated to contain the new forecast values.
 * @throws ModelNotInitializedException if getMSE is called before the
 *         model has been initialized with a call to init.
 */
public DataSet forecast( DataSet dataSet )
{
    if ( !initialized )
        throw new ModelNotInitializedException();
    
    Iterator<DataPoint> it = dataSet.iterator();
    while ( it.hasNext() )
        {
            DataPoint dp = it.next();
            dp.setDependentValue( forecast(dp) );
        }
    
    return dataSet;        
}