Java Code Examples for org.apache.kylin.common.KylinConfig#isEmptySegmentAllowed()

The following examples show how to use org.apache.kylin.common.KylinConfig#isEmptySegmentAllowed() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: RedistributeFlatHiveTableByLivyStep.java From kylin-on-parquet-v2 with Apache License 2.0

4 votes

@Override
protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
    KylinConfig config = getCubeSpecificConfig();
    String intermediateTable = getIntermediateTable();
    String database, tableName;
    if (intermediateTable.indexOf(".") > 0) {
        database = intermediateTable.substring(0, intermediateTable.indexOf("."));
        tableName = intermediateTable.substring(intermediateTable.indexOf(".") + 1);
    } else {
        database = config.getHiveDatabaseForIntermediateTable();
        tableName = intermediateTable;
    }

    try {
        long rowCount = computeRowCount(database, tableName);
        logger.debug("Row count of table '" + intermediateTable + "' is " + rowCount);
        if (rowCount == 0) {
            if (!config.isEmptySegmentAllowed()) {
                stepLogger.log("Detect upstream hive table is empty, "
                        + "fail the job because \"kylin.job.allow-empty-segment\" = \"false\"");
                return new ExecuteResult(ExecuteResult.State.ERROR, stepLogger.getBufferedLog());
            } else {
                return new ExecuteResult(ExecuteResult.State.SUCCEED, "Row count is 0, no need to redistribute");
            }
        }

        int mapperInputRows = config.getHadoopJobMapperInputRows();

        int numReducers = Math.round(rowCount / ((float) mapperInputRows));
        numReducers = Math.max(1, numReducers);
        numReducers = Math.min(numReducers, config.getHadoopJobMaxReducerNumber());

        stepLogger.log("total input rows = " + rowCount);
        stepLogger.log("expected input rows per mapper = " + mapperInputRows);
        stepLogger.log("num reducers for RedistributeFlatHiveTableStep = " + numReducers);

        redistributeTable(config, numReducers);
        long dataSize = getDataSize(database, tableName);
        getManager().addJobInfo(getId(), ExecutableConstants.HDFS_BYTES_WRITTEN, "" + dataSize);
        return new ExecuteResult(ExecuteResult.State.SUCCEED, stepLogger.getBufferedLog());

    } catch (Exception e) {
        logger.error("job:" + getId() + " execute finished with exception", e);
        return new ExecuteResult(ExecuteResult.State.ERROR, stepLogger.getBufferedLog(), e);
    }
}

Example 2

Source File: RedistributeFlatHiveTableStep.java From kylin-on-parquet-v2 with Apache License 2.0

4 votes

@Override
protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
    KylinConfig config = getCubeSpecificConfig();
    String intermediateTable = getIntermediateTable();
    String database, tableName;
    if (intermediateTable.indexOf(".") > 0) {
        database = intermediateTable.substring(0, intermediateTable.indexOf("."));
        tableName = intermediateTable.substring(intermediateTable.indexOf(".") + 1);
    } else {
        database = config.getHiveDatabaseForIntermediateTable();
        tableName = intermediateTable;
    }

    try {
        long rowCount = computeRowCount(database, tableName);
        logger.debug("Row count of table '" + intermediateTable + "' is " + rowCount);
        if (rowCount == 0) {
            if (!config.isEmptySegmentAllowed()) {
                stepLogger.log("Detect upstream hive table is empty, "
                        + "fail the job because \"kylin.job.allow-empty-segment\" = \"false\"");
                return new ExecuteResult(ExecuteResult.State.ERROR, stepLogger.getBufferedLog());
            } else {
                return new ExecuteResult(ExecuteResult.State.SUCCEED, "Row count is 0, no need to redistribute");
            }
        }

        int mapperInputRows = config.getHadoopJobMapperInputRows();

        int numReducers = Math.round(rowCount / ((float) mapperInputRows));
        numReducers = Math.max(1, numReducers);
        numReducers = Math.min(numReducers, config.getHadoopJobMaxReducerNumber());

        stepLogger.log("total input rows = " + rowCount);
        stepLogger.log("expected input rows per mapper = " + mapperInputRows);
        stepLogger.log("num reducers for RedistributeFlatHiveTableStep = " + numReducers);

        redistributeTable(config, numReducers);
        long dataSize = getDataSize(database, tableName);
        getManager().addJobInfo(getId(), ExecutableConstants.HDFS_BYTES_WRITTEN, "" + dataSize);
        return new ExecuteResult(ExecuteResult.State.SUCCEED, stepLogger.getBufferedLog());

    } catch (Exception e) {
        logger.error("job:" + getId() + " execute finished with exception", e);
        return new ExecuteResult(ExecuteResult.State.ERROR, stepLogger.getBufferedLog(), e);
    }
}

Example 3

Source File: RedistributeFlatHiveTableByLivyStep.java From kylin with Apache License 2.0

4 votes

@Override
protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
    KylinConfig config = getCubeSpecificConfig();
    String intermediateTable = getIntermediateTable();
    String database, tableName;
    if (intermediateTable.indexOf(".") > 0) {
        database = intermediateTable.substring(0, intermediateTable.indexOf("."));
        tableName = intermediateTable.substring(intermediateTable.indexOf(".") + 1);
    } else {
        database = config.getHiveDatabaseForIntermediateTable();
        tableName = intermediateTable;
    }

    try {
        long rowCount = computeRowCount(database, tableName);
        logger.debug("Row count of table '" + intermediateTable + "' is " + rowCount);
        if (rowCount == 0) {
            if (!config.isEmptySegmentAllowed()) {
                stepLogger.log("Detect upstream hive table is empty, "
                        + "fail the job because \"kylin.job.allow-empty-segment\" = \"false\"");
                return new ExecuteResult(ExecuteResult.State.ERROR, stepLogger.getBufferedLog());
            } else {
                return new ExecuteResult(ExecuteResult.State.SUCCEED, "Row count is 0, no need to redistribute");
            }
        }

        int mapperInputRows = config.getHadoopJobMapperInputRows();

        int numReducers = Math.round(rowCount / ((float) mapperInputRows));
        numReducers = Math.max(1, numReducers);
        numReducers = Math.min(numReducers, config.getHadoopJobMaxReducerNumber());

        stepLogger.log("total input rows = " + rowCount);
        stepLogger.log("expected input rows per mapper = " + mapperInputRows);
        stepLogger.log("num reducers for RedistributeFlatHiveTableStep = " + numReducers);

        redistributeTable(config, numReducers);
        long dataSize = getDataSize(database, tableName);
        getManager().addJobInfo(getId(), ExecutableConstants.HDFS_BYTES_WRITTEN, "" + dataSize);
        return new ExecuteResult(ExecuteResult.State.SUCCEED, stepLogger.getBufferedLog());

    } catch (Exception e) {
        logger.error("job:" + getId() + " execute finished with exception", e);
        return new ExecuteResult(ExecuteResult.State.ERROR, stepLogger.getBufferedLog(), e);
    }
}

Example 4

Source File: RedistributeFlatHiveTableStep.java From kylin with Apache License 2.0

4 votes

@Override
protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
    KylinConfig config = getCubeSpecificConfig();
    String intermediateTable = getIntermediateTable();
    String database, tableName;
    if (intermediateTable.indexOf(".") > 0) {
        database = intermediateTable.substring(0, intermediateTable.indexOf("."));
        tableName = intermediateTable.substring(intermediateTable.indexOf(".") + 1);
    } else {
        database = config.getHiveDatabaseForIntermediateTable();
        tableName = intermediateTable;
    }

    try {
        long rowCount = computeRowCount(database, tableName);
        logger.debug("Row count of table '" + intermediateTable + "' is " + rowCount);
        if (rowCount == 0) {
            if (!config.isEmptySegmentAllowed()) {
                stepLogger.log("Detect upstream hive table is empty, "
                        + "fail the job because \"kylin.job.allow-empty-segment\" = \"false\"");
                return new ExecuteResult(ExecuteResult.State.ERROR, stepLogger.getBufferedLog());
            } else {
                return new ExecuteResult(ExecuteResult.State.SUCCEED, "Row count is 0, no need to redistribute");
            }
        }

        int mapperInputRows = config.getHadoopJobMapperInputRows();

        int numReducers = Math.round(rowCount / ((float) mapperInputRows));
        numReducers = Math.max(1, numReducers);
        numReducers = Math.min(numReducers, config.getHadoopJobMaxReducerNumber());

        stepLogger.log("total input rows = " + rowCount);
        stepLogger.log("expected input rows per mapper = " + mapperInputRows);
        stepLogger.log("num reducers for RedistributeFlatHiveTableStep = " + numReducers);

        redistributeTable(config, numReducers);
        long dataSize = getDataSize(database, tableName);
        getManager().addJobInfo(getId(), ExecutableConstants.HDFS_BYTES_WRITTEN, "" + dataSize);
        return new ExecuteResult(ExecuteResult.State.SUCCEED, stepLogger.getBufferedLog());

    } catch (Exception e) {
        logger.error("job:" + getId() + " execute finished with exception", e);
        return new ExecuteResult(ExecuteResult.State.ERROR, stepLogger.getBufferedLog(), e);
    }
}