Java Code Examples for org.apache.kylin.job.dao.ExecutablePO#getLastModified()

The following examples show how to use org.apache.kylin.job.dao.ExecutablePO#getLastModified() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: KylinHealthCheckJob.java From kylin-on-parquet-v2 with Apache License 2.0

6 votes

private void checkStoppedJob() throws Exception {
    reporter.log("## Cleanup stopped job");
    int staleJobThresholdInDays = config.getStaleJobThresholdInDays();
    long outdatedJobTimeCut = System.currentTimeMillis() - 1L * staleJobThresholdInDays * 24 * 60 * 60 * 1000;
    ExecutableDao executableDao = ExecutableDao.getInstance(config);
    // discard all expired ERROR or STOPPED jobs
    List<ExecutablePO> allExecutable = executableDao.getJobs();
    for (ExecutablePO executable : allExecutable) {
        long lastModified = executable.getLastModified();
        String jobStatus = executableDao.getJobOutput(executable.getUuid()).getStatus();
        if (lastModified < outdatedJobTimeCut && (ExecutableState.ERROR.toString().equals(jobStatus)
                || ExecutableState.STOPPED.toString().equals(jobStatus))) {
            // ExecutableManager.getInstance(config).discardJob(executable.getId());
            if (executable.getType().equals(CubingJob.class.getName())
                    || executable.getType().equals(CheckpointExecutable.class.getName())) {
                reporter.log("Should discard job: {}, which in ERROR/STOPPED state for {} days", executable.getId(),
                        staleJobThresholdInDays);
            } else {
                logger.warn("Unknown out of date job: {} with type: {}, which in ERROR/STOPPED state for {} days",
                        executable.getId(), executable.getType(), staleJobThresholdInDays);
            }
        }
    }
}

Example 2

Source File: KylinHealthCheckJob.java From kylin with Apache License 2.0

6 votes

private void checkStoppedJob() throws Exception {
    reporter.log("## Cleanup stopped job");
    int staleJobThresholdInDays = config.getStaleJobThresholdInDays();
    long outdatedJobTimeCut = System.currentTimeMillis() - 1L * staleJobThresholdInDays * 24 * 60 * 60 * 1000;
    ExecutableDao executableDao = ExecutableDao.getInstance(config);
    // discard all expired ERROR or STOPPED jobs
    List<ExecutablePO> allExecutable = executableDao.getJobs();
    for (ExecutablePO executable : allExecutable) {
        long lastModified = executable.getLastModified();
        String jobStatus = executableDao.getJobOutput(executable.getUuid()).getStatus();
        if (lastModified < outdatedJobTimeCut && (ExecutableState.ERROR.toString().equals(jobStatus)
                || ExecutableState.STOPPED.toString().equals(jobStatus))) {
            // ExecutableManager.getInstance(config).discardJob(executable.getId());
            if (executable.getType().equals(CubingJob.class.getName())
                    || executable.getType().equals(CheckpointExecutable.class.getName())) {
                reporter.log("Should discard job: {}, which in ERROR/STOPPED state for {} days", executable.getId(),
                        staleJobThresholdInDays);
            } else {
                logger.warn("Unknown out of date job: {} with type: {}, which in ERROR/STOPPED state for {} days",
                        executable.getId(), executable.getType(), staleJobThresholdInDays);
            }
        }
    }
}

Example 3

Source File: MetadataCleanupJob.java From kylin-on-parquet-v2 with Apache License 2.0

4 votes

public void cleanup() throws Exception {
    CubeManager cubeManager = CubeManager.getInstance(config);

    List<String> toDeleteResource = Lists.newArrayList();

    // two level resources, snapshot tables and cube statistics
    for (String resourceRoot : new String[] { ResourceStore.SNAPSHOT_RESOURCE_ROOT, ResourceStore.CUBE_STATISTICS_ROOT }) {
        NavigableSet<String> snapshotTables = getStore().listResources(resourceRoot);

        if (snapshotTables != null) {
            for (String snapshotTable : snapshotTables) {
                NavigableSet<String> snapshotNames = getStore().listResources(snapshotTable);
                if (snapshotNames != null)
                    for (String snapshot : snapshotNames) {
                        if (isOlderThanThreshold(getStore().getResourceTimestamp(snapshot)))
                            toDeleteResource.add(snapshot);
                    }
            }
        }
    }

    // three level resources, only dictionaries
    NavigableSet<String> dictTables = getStore().listResources(ResourceStore.DICT_RESOURCE_ROOT);

    if (dictTables != null) {
        for (String table : dictTables) {
            NavigableSet<String> tableColNames = getStore().listResources(table);
            if (tableColNames != null)
                for (String tableCol : tableColNames) {
                    NavigableSet<String> dictionaries = getStore().listResources(tableCol);
                    if (dictionaries != null)
                        for (String dict : dictionaries)
                            if (isOlderThanThreshold(getStore().getResourceTimestamp(dict)))
                                toDeleteResource.add(dict);
                }
        }
    }

    Set<String> activeResourceList = Sets.newHashSet();
    for (org.apache.kylin.cube.CubeInstance cube : cubeManager.listAllCubes()) {
        for (org.apache.kylin.cube.CubeSegment segment : cube.getSegments()) {
            activeResourceList.addAll(segment.getSnapshotPaths());
            activeResourceList.addAll(segment.getDictionaryPaths());
            activeResourceList.add(segment.getStatisticsResourcePath());
        }
    }

    toDeleteResource.removeAll(activeResourceList);

    // delete old and completed jobs
    ExecutableDao executableDao = ExecutableDao.getInstance(KylinConfig.getInstanceFromEnv());
    List<ExecutablePO> allExecutable = executableDao.getJobs();
    for (ExecutablePO executable : allExecutable) {
        long lastModified = executable.getLastModified();
        ExecutableOutputPO output = executableDao.getJobOutput(executable.getUuid());
        if (System.currentTimeMillis() - lastModified > TIME_THREADSHOLD_FOR_JOB && (ExecutableState.SUCCEED.toString().equals(output.getStatus()) || ExecutableState.DISCARDED.toString().equals(output.getStatus()))) {
            toDeleteResource.add(ResourceStore.EXECUTE_RESOURCE_ROOT + "/" + executable.getUuid());
            toDeleteResource.add(ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT + "/" + executable.getUuid());

            for (ExecutablePO task : executable.getTasks()) {
                toDeleteResource.add(ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT + "/" + task.getUuid());
            }
        }
    }

    if (toDeleteResource.size() > 0) {
        logger.info("The following resources have no reference or is too old, will be cleaned from metadata store: \n");

        for (String s : toDeleteResource) {
            logger.info(s);
            if (delete == true) {
                getStore().deleteResource(s);
            }
        }
    } else {
        logger.info("No resource to be cleaned up from metadata store;");
    }

}

Example 4

Source File: MetadataCleanupJob.java From kylin with Apache License 2.0

4 votes

public void cleanup() throws Exception {
    CubeManager cubeManager = CubeManager.getInstance(config);

    List<String> toDeleteResource = Lists.newArrayList();

    // two level resources, snapshot tables and cube statistics
    for (String resourceRoot : new String[] { ResourceStore.SNAPSHOT_RESOURCE_ROOT, ResourceStore.CUBE_STATISTICS_ROOT }) {
        NavigableSet<String> snapshotTables = getStore().listResources(resourceRoot);

        if (snapshotTables != null) {
            for (String snapshotTable : snapshotTables) {
                NavigableSet<String> snapshotNames = getStore().listResources(snapshotTable);
                if (snapshotNames != null)
                    for (String snapshot : snapshotNames) {
                        if (isOlderThanThreshold(getStore().getResourceTimestamp(snapshot)))
                            toDeleteResource.add(snapshot);
                    }
            }
        }
    }

    // three level resources, only dictionaries
    NavigableSet<String> dictTables = getStore().listResources(ResourceStore.DICT_RESOURCE_ROOT);

    if (dictTables != null) {
        for (String table : dictTables) {
            NavigableSet<String> tableColNames = getStore().listResources(table);
            if (tableColNames != null)
                for (String tableCol : tableColNames) {
                    NavigableSet<String> dictionaries = getStore().listResources(tableCol);
                    if (dictionaries != null)
                        for (String dict : dictionaries)
                            if (isOlderThanThreshold(getStore().getResourceTimestamp(dict)))
                                toDeleteResource.add(dict);
                }
        }
    }

    Set<String> activeResourceList = Sets.newHashSet();
    for (org.apache.kylin.cube.CubeInstance cube : cubeManager.listAllCubes()) {
        for (org.apache.kylin.cube.CubeSegment segment : cube.getSegments()) {
            activeResourceList.addAll(segment.getSnapshotPaths());
            activeResourceList.addAll(segment.getDictionaryPaths());
            activeResourceList.add(segment.getStatisticsResourcePath());
        }
    }

    toDeleteResource.removeAll(activeResourceList);

    // delete old and completed jobs
    ExecutableDao executableDao = ExecutableDao.getInstance(KylinConfig.getInstanceFromEnv());
    List<ExecutablePO> allExecutable = executableDao.getJobs();
    for (ExecutablePO executable : allExecutable) {
        long lastModified = executable.getLastModified();
        ExecutableOutputPO output = executableDao.getJobOutput(executable.getUuid());
        if (System.currentTimeMillis() - lastModified > TIME_THREADSHOLD_FOR_JOB && (ExecutableState.SUCCEED.toString().equals(output.getStatus()) || ExecutableState.DISCARDED.toString().equals(output.getStatus()))) {
            toDeleteResource.add(ResourceStore.EXECUTE_RESOURCE_ROOT + "/" + executable.getUuid());
            toDeleteResource.add(ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT + "/" + executable.getUuid());

            for (ExecutablePO task : executable.getTasks()) {
                toDeleteResource.add(ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT + "/" + task.getUuid());
            }
        }
    }

    if (toDeleteResource.size() > 0) {
        logger.info("The following resources have no reference or is too old, will be cleaned from metadata store: \n");

        for (String s : toDeleteResource) {
            logger.info(s);
            if (delete == true) {
                getStore().deleteResource(s);
            }
        }
    } else {
        logger.info("No resource to be cleaned up from metadata store;");
    }

}