Java Code Examples for org.apache.hadoop.mapred.Reporter#getCounter()

The following examples show how to use org.apache.hadoop.mapred.Reporter#getCounter() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: XGBoostTrainUDTF.java From incubator-hivemall with Apache License 2.0

6 votes

@Nonnull
private static Booster train(@Nonnull final DMatrix dtrain, @Nonnegative final int round,
        @Nonnull final Map<String, Object> params, @Nullable final Reporter reporter)
        throws NoSuchMethodException, IllegalAccessException, InvocationTargetException,
        InstantiationException, XGBoostError {
    final Counters.Counter iterCounter = (reporter == null) ? null
            : reporter.getCounter("hivemall.XGBoostTrainUDTF$Counter", "iteration");

    final Booster booster = XGBoostUtils.createBooster(dtrain, params);
    for (int iter = 0; iter < round; iter++) {
        reportProgress(reporter);
        setCounterValue(iterCounter, iter + 1);

        booster.update(dtrain, iter);
    }
    return booster;
}

Example 2

Source File: ReportingUtils.java From elasticsearch-hadoop with Apache License 2.0

5 votes

private static void oldApiCounter(Reporter reporter, Enum<?> counter, long value) {
    try {
        org.apache.hadoop.mapred.Counters.Counter c = reporter.getCounter(counter);
        if (c != null) {
            c.increment(value);
        }
    } catch (Exception ex) {
        // counter unavailable
    }
}

Example 3

Source File: XGBoostTrainUDTF.java From incubator-hivemall with Apache License 2.0

4 votes

@Nonnull
private static Booster train(@Nonnull final DMatrix dtrain, @Nonnull final DMatrix dtest,
        @Nonnegative final int round, @Nonnegative final int earlyStoppingRounds,
        @Nonnull final Map<String, Object> params, @Nullable final Reporter reporter)
        throws NoSuchMethodException, IllegalAccessException, InvocationTargetException,
        InstantiationException, XGBoostError {
    final Counters.Counter iterCounter = (reporter == null) ? null
            : reporter.getCounter("hivemall.XGBoostTrainUDTF$Counter", "iteration");

    final Booster booster = XGBoostUtils.createBooster(dtrain, params);

    final boolean maximizeEvaluationMetrics =
            OptionUtils.getBoolean(params, "maximize_evaluation_metrics");
    float bestScore = maximizeEvaluationMetrics ? -Float.MAX_VALUE : Float.MAX_VALUE;
    int bestIteration = 0;

    final float[] metricsOut = new float[1];
    for (int iter = 0; iter < round; iter++) {
        reportProgress(reporter);
        setCounterValue(iterCounter, iter + 1);

        booster.update(dtrain, iter);

        String evalInfo =
                booster.evalSet(new DMatrix[] {dtest}, new String[] {"test"}, iter, metricsOut);
        logger.info(evalInfo);

        final float score = metricsOut[0];
        if (maximizeEvaluationMetrics) {
            // Update best score if the current score is better (no update when equal)
            if (score > bestScore) {
                bestScore = score;
                bestIteration = iter;
            }
        } else {
            if (score < bestScore) {
                bestScore = score;
                bestIteration = iter;
            }
        }

        if (shouldEarlyStop(earlyStoppingRounds, iter, bestIteration)) {
            logger.info(
                String.format("early stopping after %d rounds away from the best iteration",
                    earlyStoppingRounds));
            break;
        }
    }

    return booster;
}