org.apache.kylin.job.dao.ExecutableDao Java Examples

The following examples show how to use org.apache.kylin.job.dao.ExecutableDao. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: YarnLogExtractor.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up 
public void extractYarnLogAndMRJob(String jobId, File yarnLogDir) throws Exception {
    logger.info("Collecting Yarn logs and MR counters for the Job {}", jobId);
    kylinConfig = KylinConfig.getInstanceFromEnv();
    executableDao = ExecutableDao.getInstance(kylinConfig);
    ExecutablePO executablePO = null;
    executablePO = executableDao.getJob(jobId);

    if (executablePO == null) {
        logger.error("Can not find executablePO.");
        return;
    }

    for (ExecutablePO task : executablePO.getTasks()) {
        yarnLogsResources.add(task.getUuid());
    }

    for (String stepId : yarnLogsResources) {
        logger.info("Checking step {}", stepId);
        extractYarnLog(stepId, new File(yarnLogDir, stepId));
        extractMRJob(stepId, new File(yarnLogDir, stepId));
    }
}
Example #2
Source File: JobControllerTest.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up 
@Before
public void setup() throws Exception {
    super.setup();

    jobSchedulerController = new JobController();
    jobSchedulerController.setJobService(jobService);
    cubeController = new CubeController();
    cubeController.setJobService(jobService);
    cubeController.setCubeService(cubeService);

    KylinConfig testConfig = getTestConfig();
    cubeManager = CubeManager.getInstance(testConfig);
    cubeDescManager = CubeDescManager.getInstance(testConfig);
    executableDAO = ExecutableDao.getInstance(testConfig);

}
Example #3
Source File: MetadataCleanupJob.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up 
private boolean isJobComplete(ExecutableDao executableDao, ExecutablePO job) {
    String jobId = job.getUuid();
    boolean isComplete = false;
    try {
        ExecutableOutputPO output = executableDao.getJobOutput(jobId);
        String status = output.getStatus();
        String jobType = job.getType();
        if (jobType.equals(CubingJob.class.getName())
                || jobType.equals(CheckpointExecutable.class.getName())) {
            if (StringUtils.equals(status, ExecutableState.SUCCEED.toString())
                    || StringUtils.equals(status, ExecutableState.DISCARDED.toString())) {
                isComplete = true;
            }
        } else if (jobType.equals(CardinalityExecutable.class.getName())) {
            // Ignore state of DefaultChainedExecutable
            isComplete = true;
        }
    } catch (PersistentException e) {
        logger.error("Get job output failed for job uuid: {}", jobId, e);
        isComplete = true; // job output broken --> will be treat as complete
    }

    return isComplete;
}
Example #4
Source File: KylinHealthCheckJob.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up 
private void checkStoppedJob() throws Exception {
    reporter.log("## Cleanup stopped job");
    int staleJobThresholdInDays = config.getStaleJobThresholdInDays();
    long outdatedJobTimeCut = System.currentTimeMillis() - 1L * staleJobThresholdInDays * 24 * 60 * 60 * 1000;
    ExecutableDao executableDao = ExecutableDao.getInstance(config);
    // discard all expired ERROR or STOPPED jobs
    List<ExecutablePO> allExecutable = executableDao.getJobs();
    for (ExecutablePO executable : allExecutable) {
        long lastModified = executable.getLastModified();
        String jobStatus = executableDao.getJobOutput(executable.getUuid()).getStatus();
        if (lastModified < outdatedJobTimeCut && (ExecutableState.ERROR.toString().equals(jobStatus)
                || ExecutableState.STOPPED.toString().equals(jobStatus))) {
            // ExecutableManager.getInstance(config).discardJob(executable.getId());
            if (executable.getType().equals(CubingJob.class.getName())
                    || executable.getType().equals(CheckpointExecutable.class.getName())) {
                reporter.log("Should discard job: {}, which in ERROR/STOPPED state for {} days", executable.getId(),
                        staleJobThresholdInDays);
            } else {
                logger.warn("Unknown out of date job: {} with type: {}, which in ERROR/STOPPED state for {} days",
                        executable.getId(), executable.getType(), staleJobThresholdInDays);
            }
        }
    }
}
Example #5
Source File: YarnLogExtractor.java    From kylin with Apache License 2.0 6 votes vote down vote up 
public void extractYarnLogAndMRJob(String jobId, File yarnLogDir) throws Exception {
    logger.info("Collecting Yarn logs and MR counters for the Job {}", jobId);
    kylinConfig = KylinConfig.getInstanceFromEnv();
    executableDao = ExecutableDao.getInstance(kylinConfig);
    ExecutablePO executablePO = null;
    executablePO = executableDao.getJob(jobId);

    if (executablePO == null) {
        logger.error("Can not find executablePO.");
        return;
    }

    for (ExecutablePO task : executablePO.getTasks()) {
        yarnLogsResources.add(task.getUuid());
    }

    for (String stepId : yarnLogsResources) {
        logger.info("Checking step {}", stepId);
        extractYarnLog(stepId, new File(yarnLogDir, stepId));
        extractMRJob(stepId, new File(yarnLogDir, stepId));
    }
}
Example #6
Source File: JobControllerTest.java    From kylin with Apache License 2.0 6 votes vote down vote up 
@Before
public void setup() throws Exception {
    super.setup();

    jobSchedulerController = new JobController();
    jobSchedulerController.setJobService(jobService);
    cubeController = new CubeController();
    cubeController.setJobService(jobService);
    cubeController.setCubeService(cubeService);

    KylinConfig testConfig = getTestConfig();
    cubeManager = CubeManager.getInstance(testConfig);
    cubeDescManager = CubeDescManager.getInstance(testConfig);
    executableDAO = ExecutableDao.getInstance(testConfig);

}
Example #7
Source File: MetadataCleanupJob.java    From kylin with Apache License 2.0 6 votes vote down vote up 
private boolean isJobComplete(ExecutableDao executableDao, ExecutablePO job) {
    String jobId = job.getUuid();
    boolean isComplete = false;
    try {
        ExecutableOutputPO output = executableDao.getJobOutput(jobId);
        String status = output.getStatus();
        String jobType = job.getType();
        if (jobType.equals(CubingJob.class.getName())
                || jobType.equals(CheckpointExecutable.class.getName())) {
            if (StringUtils.equals(status, ExecutableState.SUCCEED.toString())
                    || StringUtils.equals(status, ExecutableState.DISCARDED.toString())) {
                isComplete = true;
            }
        } else if (jobType.equals(CardinalityExecutable.class.getName())) {
            // Ignore state of DefaultChainedExecutable
            isComplete = true;
        }
    } catch (PersistentException e) {
        logger.error("Get job output failed for job uuid: {}", jobId, e);
        isComplete = true; // job output broken --> will be treat as complete
    }

    return isComplete;
}
Example #8
Source File: KylinHealthCheckJob.java    From kylin with Apache License 2.0 6 votes vote down vote up 
private void checkStoppedJob() throws Exception {
    reporter.log("## Cleanup stopped job");
    int staleJobThresholdInDays = config.getStaleJobThresholdInDays();
    long outdatedJobTimeCut = System.currentTimeMillis() - 1L * staleJobThresholdInDays * 24 * 60 * 60 * 1000;
    ExecutableDao executableDao = ExecutableDao.getInstance(config);
    // discard all expired ERROR or STOPPED jobs
    List<ExecutablePO> allExecutable = executableDao.getJobs();
    for (ExecutablePO executable : allExecutable) {
        long lastModified = executable.getLastModified();
        String jobStatus = executableDao.getJobOutput(executable.getUuid()).getStatus();
        if (lastModified < outdatedJobTimeCut && (ExecutableState.ERROR.toString().equals(jobStatus)
                || ExecutableState.STOPPED.toString().equals(jobStatus))) {
            // ExecutableManager.getInstance(config).discardJob(executable.getId());
            if (executable.getType().equals(CubingJob.class.getName())
                    || executable.getType().equals(CheckpointExecutable.class.getName())) {
                reporter.log("Should discard job: {}, which in ERROR/STOPPED state for {} days", executable.getId(),
                        staleJobThresholdInDays);
            } else {
                logger.warn("Unknown out of date job: {} with type: {}, which in ERROR/STOPPED state for {} days",
                        executable.getId(), executable.getType(), staleJobThresholdInDays);
            }
        }
    }
}
Example #9
Source File: JobControllerTest.java    From Kylin with Apache License 2.0 6 votes vote down vote up 
@Before
public void setup() throws Exception {
    super.setUp();

    jobSchedulerController = new JobController();
    jobSchedulerController.setJobService(jobService);
    cubeController = new CubeController();
    cubeController.setJobService(jobService);
    cubeController.setCubeService(cubeService);
    KylinConfig testConfig = getTestConfig();
    cubeManager = CubeManager.getInstance(testConfig);
    cubeDescManager = CubeDescManager.getInstance(testConfig);

    executableDAO = ExecutableDao.getInstance(testConfig);

}
Example #10
Source File: JobDiagnosisInfoCLI.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
public JobDiagnosisInfoCLI() {
    super();

    packageType = "job";

    options.addOption(OPTION_JOB_ID);
    options.addOption(OPTION_INCLUDE_CUBE);
    options.addOption(OPTION_INCLUDE_CLIENT);
    options.addOption(OPTION_INCLUDE_YARN_LOGS);
    options.addOption(OPTION_INCLUDE_CONF);

    kylinConfig = KylinConfig.getInstanceFromEnv();
    executableDao = ExecutableDao.getInstance(kylinConfig);
}
Example #11
Source File: JobDiagnosisInfoCLI.java    From kylin with Apache License 2.0 5 votes vote down vote up 
public JobDiagnosisInfoCLI() {
    super();

    packageType = "job";

    options.addOption(OPTION_JOB_ID);
    options.addOption(OPTION_INCLUDE_CUBE);
    options.addOption(OPTION_INCLUDE_CLIENT);
    options.addOption(OPTION_INCLUDE_YARN_LOGS);
    options.addOption(OPTION_INCLUDE_CONF);

    kylinConfig = KylinConfig.getInstanceFromEnv();
    executableDao = ExecutableDao.getInstance(kylinConfig);
}
Example #12
Source File: MetadataCleanupJob.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up 
public void cleanup() throws Exception {
    CubeManager cubeManager = CubeManager.getInstance(config);

    List<String> toDeleteResource = Lists.newArrayList();

    // two level resources, snapshot tables and cube statistics
    for (String resourceRoot : new String[] { ResourceStore.SNAPSHOT_RESOURCE_ROOT, ResourceStore.CUBE_STATISTICS_ROOT }) {
        NavigableSet<String> snapshotTables = getStore().listResources(resourceRoot);

        if (snapshotTables != null) {
            for (String snapshotTable : snapshotTables) {
                NavigableSet<String> snapshotNames = getStore().listResources(snapshotTable);
                if (snapshotNames != null)
                    for (String snapshot : snapshotNames) {
                        if (isOlderThanThreshold(getStore().getResourceTimestamp(snapshot)))
                            toDeleteResource.add(snapshot);
                    }
            }
        }
    }

    // three level resources, only dictionaries
    NavigableSet<String> dictTables = getStore().listResources(ResourceStore.DICT_RESOURCE_ROOT);

    if (dictTables != null) {
        for (String table : dictTables) {
            NavigableSet<String> tableColNames = getStore().listResources(table);
            if (tableColNames != null)
                for (String tableCol : tableColNames) {
                    NavigableSet<String> dictionaries = getStore().listResources(tableCol);
                    if (dictionaries != null)
                        for (String dict : dictionaries)
                            if (isOlderThanThreshold(getStore().getResourceTimestamp(dict)))
                                toDeleteResource.add(dict);
                }
        }
    }

    Set<String> activeResourceList = Sets.newHashSet();
    for (org.apache.kylin.cube.CubeInstance cube : cubeManager.listAllCubes()) {
        for (org.apache.kylin.cube.CubeSegment segment : cube.getSegments()) {
            activeResourceList.addAll(segment.getSnapshotPaths());
            activeResourceList.addAll(segment.getDictionaryPaths());
            activeResourceList.add(segment.getStatisticsResourcePath());
        }
    }

    toDeleteResource.removeAll(activeResourceList);

    // delete old and completed jobs
    ExecutableDao executableDao = ExecutableDao.getInstance(KylinConfig.getInstanceFromEnv());
    List<ExecutablePO> allExecutable = executableDao.getJobs();
    for (ExecutablePO executable : allExecutable) {
        long lastModified = executable.getLastModified();
        ExecutableOutputPO output = executableDao.getJobOutput(executable.getUuid());
        if (System.currentTimeMillis() - lastModified > TIME_THREADSHOLD_FOR_JOB && (ExecutableState.SUCCEED.toString().equals(output.getStatus()) || ExecutableState.DISCARDED.toString().equals(output.getStatus()))) {
            toDeleteResource.add(ResourceStore.EXECUTE_RESOURCE_ROOT + "/" + executable.getUuid());
            toDeleteResource.add(ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT + "/" + executable.getUuid());

            for (ExecutablePO task : executable.getTasks()) {
                toDeleteResource.add(ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT + "/" + task.getUuid());
            }
        }
    }

    if (toDeleteResource.size() > 0) {
        logger.info("The following resources have no reference or is too old, will be cleaned from metadata store: \n");

        for (String s : toDeleteResource) {
            logger.info(s);
            if (delete == true) {
                getStore().deleteResource(s);
            }
        }
    } else {
        logger.info("No resource to be cleaned up from metadata store;");
    }

}
Example #13
Source File: ExecutableManager.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up 
private ExecutableManager(KylinConfig config) {
    logger.info("Using metadata url: " + config);
    this.config = config;
    this.executableDao = ExecutableDao.getInstance(config);
}
Example #14
Source File: MetadataCleanupJob.java    From kylin with Apache License 2.0 4 votes vote down vote up 
public void cleanup() throws Exception {
    CubeManager cubeManager = CubeManager.getInstance(config);

    List<String> toDeleteResource = Lists.newArrayList();

    // two level resources, snapshot tables and cube statistics
    for (String resourceRoot : new String[] { ResourceStore.SNAPSHOT_RESOURCE_ROOT, ResourceStore.CUBE_STATISTICS_ROOT }) {
        NavigableSet<String> snapshotTables = getStore().listResources(resourceRoot);

        if (snapshotTables != null) {
            for (String snapshotTable : snapshotTables) {
                NavigableSet<String> snapshotNames = getStore().listResources(snapshotTable);
                if (snapshotNames != null)
                    for (String snapshot : snapshotNames) {
                        if (isOlderThanThreshold(getStore().getResourceTimestamp(snapshot)))
                            toDeleteResource.add(snapshot);
                    }
            }
        }
    }

    // three level resources, only dictionaries
    NavigableSet<String> dictTables = getStore().listResources(ResourceStore.DICT_RESOURCE_ROOT);

    if (dictTables != null) {
        for (String table : dictTables) {
            NavigableSet<String> tableColNames = getStore().listResources(table);
            if (tableColNames != null)
                for (String tableCol : tableColNames) {
                    NavigableSet<String> dictionaries = getStore().listResources(tableCol);
                    if (dictionaries != null)
                        for (String dict : dictionaries)
                            if (isOlderThanThreshold(getStore().getResourceTimestamp(dict)))
                                toDeleteResource.add(dict);
                }
        }
    }

    Set<String> activeResourceList = Sets.newHashSet();
    for (org.apache.kylin.cube.CubeInstance cube : cubeManager.listAllCubes()) {
        for (org.apache.kylin.cube.CubeSegment segment : cube.getSegments()) {
            activeResourceList.addAll(segment.getSnapshotPaths());
            activeResourceList.addAll(segment.getDictionaryPaths());
            activeResourceList.add(segment.getStatisticsResourcePath());
        }
    }

    toDeleteResource.removeAll(activeResourceList);

    // delete old and completed jobs
    ExecutableDao executableDao = ExecutableDao.getInstance(KylinConfig.getInstanceFromEnv());
    List<ExecutablePO> allExecutable = executableDao.getJobs();
    for (ExecutablePO executable : allExecutable) {
        long lastModified = executable.getLastModified();
        ExecutableOutputPO output = executableDao.getJobOutput(executable.getUuid());
        if (System.currentTimeMillis() - lastModified > TIME_THREADSHOLD_FOR_JOB && (ExecutableState.SUCCEED.toString().equals(output.getStatus()) || ExecutableState.DISCARDED.toString().equals(output.getStatus()))) {
            toDeleteResource.add(ResourceStore.EXECUTE_RESOURCE_ROOT + "/" + executable.getUuid());
            toDeleteResource.add(ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT + "/" + executable.getUuid());

            for (ExecutablePO task : executable.getTasks()) {
                toDeleteResource.add(ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT + "/" + task.getUuid());
            }
        }
    }

    if (toDeleteResource.size() > 0) {
        logger.info("The following resources have no reference or is too old, will be cleaned from metadata store: \n");

        for (String s : toDeleteResource) {
            logger.info(s);
            if (delete == true) {
                getStore().deleteResource(s);
            }
        }
    } else {
        logger.info("No resource to be cleaned up from metadata store;");
    }

}
Example #15
Source File: ExecutableManager.java    From kylin with Apache License 2.0 4 votes vote down vote up 
private ExecutableManager(KylinConfig config) {
    logger.info("Using metadata url: " + config);
    this.config = config;
    this.executableDao = ExecutableDao.getInstance(config);
}
Example #16
Source File: ExecutableManager.java    From Kylin with Apache License 2.0 4 votes vote down vote up 
private ExecutableManager(KylinConfig config) {
    logger.info("Using metadata url: " + config);
    this.config = config;
    this.executableDao = ExecutableDao.getInstance(config);
}