Java Code Examples for org.apache.kylin.job.execution.ExecutableState#READY

The following examples show how to use org.apache.kylin.job.execution.ExecutableState#READY . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: JobService.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up 
private ExecutableState parseToExecutableState(JobStatusEnum status) {
    Message msg = MsgPicker.getMsg();

    switch (status) {
    case DISCARDED:
        return ExecutableState.DISCARDED;
    case ERROR:
        return ExecutableState.ERROR;
    case FINISHED:
        return ExecutableState.SUCCEED;
    case NEW:
        return ExecutableState.READY;
    case PENDING:
        return ExecutableState.READY;
    case RUNNING:
        return ExecutableState.RUNNING;
    case STOPPED:
        return ExecutableState.STOPPED;
    default:
        throw new BadRequestException(String.format(Locale.ROOT, msg.getILLEGAL_EXECUTABLE_STATE(), status));
    }
}
Example 2
Source File: JobService.java    From kylin with Apache License 2.0 6 votes vote down vote up 
private ExecutableState parseToExecutableState(JobStatusEnum status) {
    Message msg = MsgPicker.getMsg();

    switch (status) {
    case DISCARDED:
        return ExecutableState.DISCARDED;
    case ERROR:
        return ExecutableState.ERROR;
    case FINISHED:
        return ExecutableState.SUCCEED;
    case NEW:
        return ExecutableState.READY;
    case PENDING:
        return ExecutableState.READY;
    case RUNNING:
        return ExecutableState.RUNNING;
    case STOPPED:
        return ExecutableState.STOPPED;
    default:
        throw new BadRequestException(String.format(Locale.ROOT, msg.getILLEGAL_EXECUTABLE_STATE(), status));
    }
}
Example 3
Source File: CubeMetadataUpgrade.java    From Kylin with Apache License 2.0 6 votes vote down vote up 
private ExecutableState parseState(JobStatusEnum state) {
    switch (state) {
        case NEW:
        case PENDING:
            return ExecutableState.READY;
        case RUNNING:
            return ExecutableState.RUNNING;
        case FINISHED:
            return ExecutableState.SUCCEED;
        case ERROR:
            return ExecutableState.ERROR;
        case DISCARDED:
            return ExecutableState.DISCARDED;
        default:
            return ExecutableState.DISCARDED;
    }
}
Example 4
Source File: CubeMetadataUpgrade.java    From Kylin with Apache License 2.0 6 votes vote down vote up 
private ExecutableState parseState(JobStepStatusEnum state) {
    switch (state) {
        case NEW:
        case PENDING:
        case WAITING:
            return ExecutableState.READY;
        case RUNNING:
            return ExecutableState.RUNNING;
        case FINISHED:
            return ExecutableState.SUCCEED;
        case ERROR:
            return ExecutableState.ERROR;
        case DISCARDED:
            return ExecutableState.DISCARDED;
        default:
            return ExecutableState.DISCARDED;
    }

}
Example 5
Source File: JobService.java    From Kylin with Apache License 2.0 6 votes vote down vote up 
private ExecutableState parseToExecutableState(JobStatusEnum status) {
    switch (status) {
        case DISCARDED:
            return ExecutableState.DISCARDED;
        case ERROR:
            return ExecutableState.ERROR;
        case FINISHED:
            return ExecutableState.SUCCEED;
        case NEW:
            return ExecutableState.READY;
        case PENDING:
            return ExecutableState.READY;
        case RUNNING:
            return ExecutableState.RUNNING;
        default:
            throw new RuntimeException("illegal status:" + status);
    }
}
Example 6
Source File: JobInfoConverterTest.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
@Test
public void testParseToJobStepStatusReturnsJobStepStatusPending() {
    ExecutableState executableState = ExecutableState.READY;
    JobStepStatusEnum jobStepStatusEnum = JobInfoConverter.parseToJobStepStatus(executableState);

    assertTrue(jobStepStatusEnum.isRunable());
    assertEquals(1, jobStepStatusEnum.getCode());
    assertEquals(JobStepStatusEnum.PENDING, jobStepStatusEnum);
}
Example 7
Source File: JobInfoConverterTest.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
@Test
public void testParseToJobStatusReturnsJobStatusPending() {
    ExecutableState executableState = ExecutableState.READY;
    JobStatusEnum jobStatusEnum = JobInfoConverter.parseToJobStatus(executableState);

    assertEquals(1, jobStatusEnum.getCode());
    assertEquals(JobStatusEnum.PENDING, jobStatusEnum);
}
Example 8
Source File: DistributedScheduler.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
private void releaseJobLock(AbstractExecutable executable) {
    if (executable instanceof DefaultChainedExecutable) {
        ExecutableState state = executable.getStatus();

        if (state != ExecutableState.READY && state != ExecutableState.RUNNING) {
            if (jobWithLocks.contains(executable.getId())) {
                logger.info(
                        executable.toString() + " will release the lock for the job: " + executable.getId());
                jobLock.unlock(getLockPath(executable.getId()));
                jobWithLocks.remove(executable.getId());
            }
        }
    }
}
Example 9
Source File: JobInfoConverterTest.java    From kylin with Apache License 2.0 5 votes vote down vote up 
@Test
public void testParseToJobStepStatusReturnsJobStepStatusPending() {
    ExecutableState executableState = ExecutableState.READY;
    JobStepStatusEnum jobStepStatusEnum = JobInfoConverter.parseToJobStepStatus(executableState);

    assertTrue(jobStepStatusEnum.isRunable());
    assertEquals(1, jobStepStatusEnum.getCode());
    assertEquals(JobStepStatusEnum.PENDING, jobStepStatusEnum);
}
Example 10
Source File: JobInfoConverterTest.java    From kylin with Apache License 2.0 5 votes vote down vote up 
@Test
public void testParseToJobStatusReturnsJobStatusPending() {
    ExecutableState executableState = ExecutableState.READY;
    JobStatusEnum jobStatusEnum = JobInfoConverter.parseToJobStatus(executableState);

    assertEquals(1, jobStatusEnum.getCode());
    assertEquals(JobStatusEnum.PENDING, jobStatusEnum);
}
Example 11
Source File: DistributedScheduler.java    From kylin with Apache License 2.0 5 votes vote down vote up 
private void releaseJobLock(AbstractExecutable executable) {
    if (executable instanceof DefaultChainedExecutable) {
        ExecutableState state = executable.getStatus();

        if (state != ExecutableState.READY && state != ExecutableState.RUNNING) {
            if (jobWithLocks.contains(executable.getId())) {
                logger.info(
                        executable.toString() + " will release the lock for the job: " + executable.getId());
                jobLock.unlock(getLockPath(executable.getId()));
                jobWithLocks.remove(executable.getId());
            }
        }
    }
}
Example 12
Source File: DefaultScheduler.java    From Kylin with Apache License 2.0 5 votes vote down vote up 
@Override
public void run() {
    // logger.debug("Job Fetcher is running...");
    Map<String, Executable> runningJobs = context.getRunningJobs();
    if (runningJobs.size() >= jobEngineConfig.getMaxConcurrentJobLimit()) {
        logger.warn("There are too many jobs running, Job Fetch will wait until next schedule time");
        return;
    }

    int nRunning = 0, nReady = 0, nOthers = 0;
    for (final String id : executableManager.getAllJobIds()) {
        if (runningJobs.containsKey(id)) {
            // logger.debug("Job id:" + id + " is already running");
            nRunning++;
            continue;
        }
        final Output output = executableManager.getOutput(id);
        if ((output.getState() != ExecutableState.READY)) {
            // logger.debug("Job id:" + id + " not runnable");
            nOthers++;
            continue;
        }
        nReady++;
        AbstractExecutable executable = executableManager.getJob(id);
        String jobDesc = executable.toString();
        logger.info(jobDesc + " prepare to schedule");
        try {
            context.addRunningJob(executable);
            jobPool.execute(new JobRunner(executable));
            logger.info(jobDesc + " scheduled");
        } catch (Exception ex) {
            context.removeRunningJob(executable);
            logger.warn(jobDesc + " fail to schedule", ex);
        }
    }
    logger.info("Job Fetcher: " + nRunning + " running, " + runningJobs.size() + " actual running, " + nReady + " ready, " + nOthers + " others");
}
Example 13
Source File: DefaultFetcherRunner.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up 
@Override
synchronized public void run() {
    try (SetThreadName ignored = new SetThreadName(//
            "FetcherRunner %s", System.identityHashCode(this))) {//
        // logger.debug("Job Fetcher is running...");
        Map<String, Executable> runningJobs = context.getRunningJobs();
        if (isJobPoolFull()) {
            return;
        }

        nRunning = 0;
        nReady = 0;
        nStopped = 0;
        nOthers = 0;
        nError = 0;
        nDiscarded = 0;
        nSUCCEED = 0;
        for (final String id : getExecutableManager().getAllJobIdsInCache()) {
            if (isJobPoolFull()) {
                return;
            }
            if (runningJobs.containsKey(id)) {
                // logger.debug("Job id:" + id + " is already running");
                nRunning++;
                continue;
            }
            if (succeedJobs.contains(id)) {
                nSUCCEED++;
                continue;
            }

            final Output outputDigest;
            try {
                outputDigest = getExecutableManager().getOutputDigest(id);
            } catch (IllegalArgumentException e) {
                logger.warn("job " + id + " output digest is null, skip.", e);
                nOthers++;
                continue;
            }
            if ((outputDigest.getState() != ExecutableState.READY)) {
                // logger.debug("Job id:" + id + " not runnable");
                jobStateCount(id);
                continue;
            }

            final AbstractExecutable executable = getExecutableManager().getJob(id);
            if (executable == null) {
                logger.info("job " + id + " get job is null, skip.");
                nOthers++;
                continue;
            }
            if (!executable.isReady()) {
                nOthers++;
                continue;
            }

            KylinConfig config = jobEngineConfig.getConfig();
            if (config.isSchedulerSafeMode()) {
                String cubeName = executable.getCubeName();
                String projectName = CubeManager.getInstance(config).getCube(cubeName).getProject();
                if (!config.getSafeModeRunnableProjects().contains(projectName) &&
                        executable.getStartTime() == 0) {
                    logger.info("New job is pending for scheduler in safe mode. Project: {}, job: {}",
                            projectName, executable.getName());
                    continue;
                }
            }

            nReady++;
            addToJobPool(executable, executable.getDefaultPriority());
        }

        fetchFailed = false;
        logger.info("Job Fetcher: " + nRunning + " should running, " + runningJobs.size() + " actual running, "
                + nStopped + " stopped, " + nReady + " ready, " + nSUCCEED + " already succeed, " + nError
                + " error, " + nDiscarded + " discarded, " + nOthers + " others");
    } catch (Throwable th) {
        fetchFailed = true; // this could happen when resource store is unavailable
        logger.warn("Job Fetcher caught a exception ", th);
    }
}
Example 14
Source File: DefaultFetcherRunner.java    From kylin with Apache License 2.0 4 votes vote down vote up 
@Override
synchronized public void run() {
    try (SetThreadName ignored = new SetThreadName(//
            "FetcherRunner %s", System.identityHashCode(this))) {//
        // logger.debug("Job Fetcher is running...");
        Map<String, Executable> runningJobs = context.getRunningJobs();
        if (isJobPoolFull()) {
            return;
        }

        nRunning = 0;
        nReady = 0;
        nStopped = 0;
        nOthers = 0;
        nError = 0;
        nDiscarded = 0;
        nSUCCEED = 0;
        for (final String id : getExecutableManager().getAllJobIdsInCache()) {
            if (isJobPoolFull()) {
                return;
            }
            if (runningJobs.containsKey(id)) {
                // logger.debug("Job id:" + id + " is already running");
                nRunning++;
                continue;
            }
            if (succeedJobs.contains(id)) {
                nSUCCEED++;
                continue;
            }

            final Output outputDigest;
            try {
                outputDigest = getExecutableManager().getOutputDigest(id);
            } catch (IllegalArgumentException e) {
                logger.warn("job " + id + " output digest is null, skip.", e);
                nOthers++;
                continue;
            }
            if ((outputDigest.getState() != ExecutableState.READY)) {
                // logger.debug("Job id:" + id + " not runnable");
                jobStateCount(id);
                continue;
            }

            final AbstractExecutable executable = getExecutableManager().getJob(id);
            if (executable == null) {
                logger.info("job " + id + " get job is null, skip.");
                nOthers++;
                continue;
            }
            if (!executable.isReady()) {
                nOthers++;
                continue;
            }

            KylinConfig config = jobEngineConfig.getConfig();
            if (config.isSchedulerSafeMode()) {
                String cubeName = executable.getCubeName();
                String projectName = CubeManager.getInstance(config).getCube(cubeName).getProject();
                if (!config.getSafeModeRunnableProjects().contains(projectName) &&
                        executable.getStartTime() == 0) {
                    logger.info("New job is pending for scheduler in safe mode. Project: {}, job: {}",
                            projectName, executable.getName());
                    continue;
                }
            }

            nReady++;
            addToJobPool(executable, executable.getDefaultPriority());
        }

        fetchFailed = false;
        logger.info("Job Fetcher: " + nRunning + " should running, " + runningJobs.size() + " actual running, "
                + nStopped + " stopped, " + nReady + " ready, " + nSUCCEED + " already succeed, " + nError
                + " error, " + nDiscarded + " discarded, " + nOthers + " others");
    } catch (Throwable th) {
        fetchFailed = true; // this could happen when resource store is unavailable
        logger.warn("Job Fetcher caught a exception ", th);
    }
}