Java Code Examples for org.apache.kylin.cube.CubeInstance#getSizeKB()

The following examples show how to use org.apache.kylin.cube.CubeInstance#getSizeKB() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: KylinHealthCheckJob.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up 
private void checkDataExpansionRate(List<CubeInstance> cubes) {
    int warningExpansionRate = config.getWarningCubeExpansionRate();
    int expansionCheckMinCubeSizeInGb = config.getExpansionCheckMinCubeSizeInGb();
    for (CubeInstance cube : cubes) {
        long sizeRecordSize = cube.getInputRecordSizeBytes();
        if (sizeRecordSize > 0) {
            long cubeDataSize = cube.getSizeKB() * 1024;
            double expansionRate = (double) cubeDataSize / sizeRecordSize;
            if (sizeRecordSize > 1L * expansionCheckMinCubeSizeInGb * 1024 * 1024 * 1024) {
                if (expansionRate > warningExpansionRate) {
                    logger.info("Cube: {} in project: {} with too large expansion rate: {}, cube data size: {}G",
                            cube.getName(), cube.getProject(), expansionRate, cubeDataSize / 1024 / 1024 / 1024);
                }
            }
        }
    }
}
Example 2
Source File: MigrationRuleSet.java    From kylin with Apache License 2.0 6 votes vote down vote up 
@Override
public void apply(Context ctx) throws RuleValidationException {
    int expansionRateThr = KylinConfig.getInstanceFromEnv().getMigrationRuleExpansionRateThreshold();

    CubeInstance cube = ctx.getCubeInstance();
    if (cube.getInputRecordSizeBytes() == 0 || cube.getSizeKB() == 0) {
        logger.warn("cube {} has zero input record size.", cube.getName());
        throw new RuleValidationException(String.format(Locale.ROOT, "Cube %s is not built.", cube.getName()));
    }
    double expansionRate = cube.getSizeKB() * 1024.0 / cube.getInputRecordSizeBytes();
    if (expansionRate > expansionRateThr) {
        logger.info(
                "cube {}, size_kb {}, cube record size {}, cube expansion rate {} larger than threshold {}.",
                cube.getName(), cube.getSizeKB(), cube.getInputRecordSizeBytes(), expansionRate,
                expansionRateThr);
        throw new RuleValidationException(
                "ExpansionRateRule: failed on expansion rate check with exceeding " + expansionRateThr);
    }
}
Example 3
Source File: KylinHealthCheckJob.java    From kylin with Apache License 2.0 6 votes vote down vote up 
private void checkDataExpansionRate(List<CubeInstance> cubes) {
    int warningExpansionRate = config.getWarningCubeExpansionRate();
    int expansionCheckMinCubeSizeInGb = config.getExpansionCheckMinCubeSizeInGb();
    for (CubeInstance cube : cubes) {
        long sizeRecordSize = cube.getInputRecordSizeBytes();
        if (sizeRecordSize > 0) {
            long cubeDataSize = cube.getSizeKB() * 1024;
            double expansionRate = (double) cubeDataSize / sizeRecordSize;
            if (sizeRecordSize > 1L * expansionCheckMinCubeSizeInGb * 1024 * 1024 * 1024) {
                if (expansionRate > warningExpansionRate) {
                    logger.info("Cube: {} in project: {} with too large expansion rate: {}, cube data size: {}G",
                            cube.getName(), cube.getProject(), expansionRate, cubeDataSize / 1024 / 1024 / 1024);
                }
            }
        }
    }
}
Example 4
Source File: DashboardService.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
public MetricsResponse getCubeMetrics(String projectName, String cubeName) {
    MetricsResponse cubeMetrics = new MetricsResponse();
    Float totalCubeSize = 0f;
    long totalRecoadSize = 0;
    List<CubeInstance> cubeInstances = cubeService.listAllCubes(cubeName, projectName, null, true);
    Integer totalCube = cubeInstances.size();
    if (projectName == null) {
        totalCube += getHybridManager().listHybridInstances().size();
    } else {
        ProjectInstance project = getProjectManager().getProject(projectName);
        totalCube += project.getRealizationCount(RealizationType.HYBRID);
    }
    Float minCubeExpansion = Float.POSITIVE_INFINITY;
    Float maxCubeExpansion = Float.NEGATIVE_INFINITY;
    cubeMetrics.increase("totalCube", totalCube.floatValue());
    for (CubeInstance cubeInstance : cubeInstances) {
        if (cubeInstance.getInputRecordSizeBytes() > 0) {
            totalCubeSize += cubeInstance.getSizeKB();
            totalRecoadSize += cubeInstance.getInputRecordSizeBytes();
            Float cubeExpansion = new Float(cubeInstance.getSizeKB()) * 1024
                    / cubeInstance.getInputRecordSizeBytes();
            if (cubeExpansion > maxCubeExpansion) {
                maxCubeExpansion = cubeExpansion;
            }
            if (cubeExpansion < minCubeExpansion) {
                minCubeExpansion = cubeExpansion;
            }
        }
    }
    Float avgCubeExpansion = 0f;
    if (totalRecoadSize != 0) {
        avgCubeExpansion = totalCubeSize * 1024 / totalRecoadSize;
    }
    cubeMetrics.increase("avgCubeExpansion", avgCubeExpansion);
    cubeMetrics.increase("maxCubeExpansion", maxCubeExpansion == Float.NEGATIVE_INFINITY ? 0 : maxCubeExpansion);
    cubeMetrics.increase("minCubeExpansion", minCubeExpansion == Float.POSITIVE_INFINITY ? 0 : minCubeExpansion);
    return cubeMetrics;
}
Example 5
Source File: DashboardService.java    From kylin with Apache License 2.0 5 votes vote down vote up 
public MetricsResponse getCubeMetrics(String projectName, String cubeName) {
    MetricsResponse cubeMetrics = new MetricsResponse();
    Float totalCubeSize = 0f;
    long totalRecoadSize = 0;
    List<CubeInstance> cubeInstances = cubeService.listAllCubes(cubeName, projectName, null, true);
    Integer totalCube = cubeInstances.size();
    if (projectName == null) {
        totalCube += getHybridManager().listHybridInstances().size();
    } else {
        ProjectInstance project = getProjectManager().getProject(projectName);
        totalCube += project.getRealizationCount(RealizationType.HYBRID);
    }
    Float minCubeExpansion = Float.POSITIVE_INFINITY;
    Float maxCubeExpansion = Float.NEGATIVE_INFINITY;
    cubeMetrics.increase("totalCube", totalCube.floatValue());
    for (CubeInstance cubeInstance : cubeInstances) {
        if (cubeInstance.getInputRecordSizeBytes() > 0) {
            totalCubeSize += cubeInstance.getSizeKB();
            totalRecoadSize += cubeInstance.getInputRecordSizeBytes();
            Float cubeExpansion = new Float(cubeInstance.getSizeKB()) * 1024
                    / cubeInstance.getInputRecordSizeBytes();
            if (cubeExpansion > maxCubeExpansion) {
                maxCubeExpansion = cubeExpansion;
            }
            if (cubeExpansion < minCubeExpansion) {
                minCubeExpansion = cubeExpansion;
            }
        }
    }
    Float avgCubeExpansion = 0f;
    if (totalRecoadSize != 0) {
        avgCubeExpansion = totalCubeSize * 1024 / totalRecoadSize;
    }
    cubeMetrics.increase("avgCubeExpansion", avgCubeExpansion);
    cubeMetrics.increase("maxCubeExpansion", maxCubeExpansion == Float.NEGATIVE_INFINITY ? 0 : maxCubeExpansion);
    cubeMetrics.increase("minCubeExpansion", minCubeExpansion == Float.POSITIVE_INFINITY ? 0 : minCubeExpansion);
    return cubeMetrics;
}