Java Code Examples for org.apache.kylin.job.execution.ExecutableState#ERROR

The following examples show how to use org.apache.kylin.job.execution.ExecutableState#ERROR . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: JobService.java    From kylin with Apache License 2.0 6 votes vote down vote up 
private ExecutableState parseToExecutableState(JobStatusEnum status) {
    Message msg = MsgPicker.getMsg();

    switch (status) {
    case DISCARDED:
        return ExecutableState.DISCARDED;
    case ERROR:
        return ExecutableState.ERROR;
    case FINISHED:
        return ExecutableState.SUCCEED;
    case NEW:
        return ExecutableState.READY;
    case PENDING:
        return ExecutableState.READY;
    case RUNNING:
        return ExecutableState.RUNNING;
    case STOPPED:
        return ExecutableState.STOPPED;
    default:
        throw new BadRequestException(String.format(Locale.ROOT, msg.getILLEGAL_EXECUTABLE_STATE(), status));
    }
}
Example 2
Source File: FetcherRunner.java    From kylin with Apache License 2.0 6 votes vote down vote up 
protected void jobStateCount(String id) {
    final Output outputDigest = getExecutableManager().getOutputDigest(id);
    // logger.debug("Job id:" + id + " not runnable");
    if (outputDigest.getState() == ExecutableState.SUCCEED) {
        succeedJobs.add(id);
        nSUCCEED++;
    } else if (outputDigest.getState() == ExecutableState.ERROR) {
        nError++;
    } else if (outputDigest.getState() == ExecutableState.DISCARDED) {
        nDiscarded++;
    } else if (outputDigest.getState() == ExecutableState.STOPPED) {
        nStopped++;
    } else {
        if (fetchFailed) {
            getExecutableManager().forceKillJob(id);
            nError++;
        } else {
            nOthers++;
        }
    }
}
Example 3
Source File: CubeMetadataUpgrade.java    From Kylin with Apache License 2.0 6 votes vote down vote up 
private ExecutableState parseState(JobStatusEnum state) {
    switch (state) {
        case NEW:
        case PENDING:
            return ExecutableState.READY;
        case RUNNING:
            return ExecutableState.RUNNING;
        case FINISHED:
            return ExecutableState.SUCCEED;
        case ERROR:
            return ExecutableState.ERROR;
        case DISCARDED:
            return ExecutableState.DISCARDED;
        default:
            return ExecutableState.DISCARDED;
    }
}
Example 4
Source File: CubingJob.java    From kylin with Apache License 2.0 6 votes vote down vote up 
protected void updateMetrics(ExecutableContext context, ExecuteResult result, ExecutableState state) {
    JobMetricsFacade.JobStatisticsResult jobStats = new JobMetricsFacade.JobStatisticsResult();
    jobStats.setWrapper(getSubmitter(), getProjectName(), CubingExecutableUtil.getCubeName(getParams()), getId(),
            getJobType(), getAlgorithm() == null ? "NULL" : getAlgorithm().toString());

    if (state == ExecutableState.SUCCEED) {
        jobStats.setJobStats(findSourceSizeBytes(), findCubeSizeBytes(), getDuration(), getMapReduceWaitTime(),
                getPerBytesTimeCost(findSourceSizeBytes(), getDuration()));
        if (CubingJobTypeEnum.getByName(getJobType()) == CubingJobTypeEnum.BUILD) {
            jobStats.setJobStepStats(getTaskDurationByName(ExecutableConstants.STEP_NAME_FACT_DISTINCT_COLUMNS),
                    getTaskDurationByName(ExecutableConstants.STEP_NAME_BUILD_DICTIONARY),
                    getTaskDurationByName(ExecutableConstants.STEP_NAME_BUILD_IN_MEM_CUBE),
                    getTaskDurationByName(ExecutableConstants.STEP_NAME_CONVERT_CUBOID_TO_HFILE));
        }
    } else if (state == ExecutableState.ERROR) {
        jobStats.setJobException(result.getThrowable() != null ? result.getThrowable() : new Exception());
    }
    JobMetricsFacade.updateMetrics(jobStats);
}
Example 5
Source File: ExecutableManager.java    From Kylin with Apache License 2.0 6 votes vote down vote up 
public void resumeJob(String jobId) {
    AbstractExecutable job = getJob(jobId);
    if (job == null) {
        return;
    }
    updateJobOutput(jobId, ExecutableState.READY, null, null);
    if (job instanceof DefaultChainedExecutable) {
        List<AbstractExecutable> tasks = ((DefaultChainedExecutable) job).getTasks();
        for (AbstractExecutable task : tasks) {
            if (task.getStatus() == ExecutableState.ERROR) {
                updateJobOutput(task.getId(), ExecutableState.READY, null, null);
                break;
            }
        }
    }
}
Example 6
Source File: FetcherRunner.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up 
protected void jobStateCount(String id) {
    final Output outputDigest = getExecutableManager().getOutputDigest(id);
    // logger.debug("Job id:" + id + " not runnable");
    if (outputDigest.getState() == ExecutableState.SUCCEED) {
        succeedJobs.add(id);
        nSUCCEED++;
    } else if (outputDigest.getState() == ExecutableState.ERROR) {
        nError++;
    } else if (outputDigest.getState() == ExecutableState.DISCARDED) {
        nDiscarded++;
    } else if (outputDigest.getState() == ExecutableState.STOPPED) {
        nStopped++;
    } else {
        if (fetchFailed) {
            getExecutableManager().forceKillJob(id);
            nError++;
        } else {
            nOthers++;
        }
    }
}
Example 7
Source File: CubingJob.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up 
protected void updateMetrics(ExecutableContext context, ExecuteResult result, ExecutableState state) {
    JobMetricsFacade.JobStatisticsResult jobStats = new JobMetricsFacade.JobStatisticsResult();
    jobStats.setWrapper(getSubmitter(), getProjectName(), CubingExecutableUtil.getCubeName(getParams()), getId(),
            getJobType(), getAlgorithm() == null ? "NULL" : getAlgorithm().toString());

    if (state == ExecutableState.SUCCEED) {
        jobStats.setJobStats(findSourceSizeBytes(), findCubeSizeBytes(), getDuration(), getMapReduceWaitTime(),
                getPerBytesTimeCost(findSourceSizeBytes(), getDuration()));
        if (CubingJobTypeEnum.getByName(getJobType()) == CubingJobTypeEnum.BUILD) {
            jobStats.setJobStepStats(getTaskDurationByName(ExecutableConstants.STEP_NAME_FACT_DISTINCT_COLUMNS),
                    getTaskDurationByName(ExecutableConstants.STEP_NAME_BUILD_DICTIONARY),
                    getTaskDurationByName(ExecutableConstants.STEP_NAME_BUILD_IN_MEM_CUBE),
                    getTaskDurationByName(ExecutableConstants.STEP_NAME_CONVERT_CUBOID_TO_HFILE));
        }
    } else if (state == ExecutableState.ERROR) {
        jobStats.setJobException(result.getThrowable() != null ? result.getThrowable() : new Exception());
    }
    JobMetricsFacade.updateMetrics(jobStats);
}
Example 8
Source File: JobService.java    From Kylin with Apache License 2.0 6 votes vote down vote up 
private ExecutableState parseToExecutableState(JobStatusEnum status) {
    switch (status) {
        case DISCARDED:
            return ExecutableState.DISCARDED;
        case ERROR:
            return ExecutableState.ERROR;
        case FINISHED:
            return ExecutableState.SUCCEED;
        case NEW:
            return ExecutableState.READY;
        case PENDING:
            return ExecutableState.READY;
        case RUNNING:
            return ExecutableState.RUNNING;
        default:
            throw new RuntimeException("illegal status:" + status);
    }
}
Example 9
Source File: JobInfoConverterTest.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
@Test
public void testParseToJobStepStatusReturnsJobStepStatusError() {
    ExecutableState executableState = ExecutableState.ERROR;
    JobStepStatusEnum jobStepStatusEnum = JobInfoConverter.parseToJobStepStatus(executableState);

    assertTrue(jobStepStatusEnum.isRunable());
    assertTrue(jobStepStatusEnum.isComplete());
    assertEquals(8, jobStepStatusEnum.getCode());
    assertEquals(JobStepStatusEnum.ERROR, jobStepStatusEnum);
}
Example 10
Source File: JobInfoConverterTest.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
@Test
public void testParseToJobStatusReturnsJobStatusError() {
    ExecutableState executableState = ExecutableState.ERROR;
    JobStatusEnum jobStatusEnum = JobInfoConverter.parseToJobStatus(executableState);

    assertEquals(8, jobStatusEnum.getCode());
    assertEquals(JobStatusEnum.ERROR, jobStatusEnum);
}
Example 11
Source File: BuildCubeWithStream.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
protected void waitForJob(String jobId) {
    while (true) {
        AbstractExecutable job = jobService.getJob(jobId);
        if (job.getStatus() == ExecutableState.SUCCEED || job.getStatus() == ExecutableState.ERROR
                || job.getStatus() == ExecutableState.DISCARDED) {
            break;
        } else {
            try {
                Thread.sleep(5000);
            } catch (InterruptedException e) {
                e.printStackTrace();
            }
        }
    }
}
Example 12
Source File: BuildCubeWithStream.java    From kylin with Apache License 2.0 5 votes vote down vote up 
protected void waitForJob(String jobId) {
    while (true) {
        AbstractExecutable job = jobService.getJob(jobId);
        if (job.getStatus() == ExecutableState.SUCCEED || job.getStatus() == ExecutableState.ERROR
                || job.getStatus() == ExecutableState.DISCARDED) {
            break;
        } else {
            try {
                Thread.sleep(5000);
            } catch (InterruptedException e) {
                e.printStackTrace();
            }
        }
    }
}
Example 13
Source File: BuildCubeWithEngine.java    From kylin with Apache License 2.0 5 votes vote down vote up 
protected ExecutableState waitForJob(String jobId) {
    while (true) {
        AbstractExecutable job = jobService.getJob(jobId);
        if (job.getStatus() == ExecutableState.SUCCEED || job.getStatus() == ExecutableState.ERROR) {
            return job.getStatus();
        } else {
            try {
                Thread.sleep(5000);
            } catch (InterruptedException e) {
                e.printStackTrace();
            }
        }
    }
}
Example 14
Source File: BuildCubeWithEngine.java    From kylin with Apache License 2.0 5 votes vote down vote up 
private boolean checkJobState() throws Exception {
    List<String> jobIds = jobService.getAllJobIdsInCache();
    while (true) {
        if (jobIds.size() == 0) {
            return true;
        }
        for (int i = jobIds.size() - 1; i >= 0; i--) {
            String jobId = jobIds.get(i);
            Output job = jobService.getOutputDigest(jobId);
            if (job.getState() == ExecutableState.ERROR) {
                return false;
            } else if (job.getState() == ExecutableState.SUCCEED) {
                String checkActionName = jobCheckActionMap.get(jobId);
                CubeSegment jobSegment = jobSegmentMap.get(jobId);
                if (checkActionName != null) {
                    Method checkAction = this.getClass().getDeclaredMethod(checkActionName, CubeSegment.class);
                    checkAction.invoke(this, jobSegment);
                    jobCheckActionMap.remove(jobId);
                    jobSegmentMap.remove(jobId);
                }
                jobIds.remove(i);
            }
        }
        try {
            Thread.sleep(5000);
        } catch (InterruptedException e) {
            e.printStackTrace();
        }
    }
}
Example 15
Source File: BaseTestDistributedScheduler.java    From kylin with Apache License 2.0 5 votes vote down vote up 
void waitForJobFinish(String jobId) {
    while (true) {
        AbstractExecutable job = execMgr.getJob(jobId);
        final ExecutableState status = job.getStatus();
        if (status == ExecutableState.SUCCEED || status == ExecutableState.ERROR || status == ExecutableState.STOPPED || status == ExecutableState.DISCARDED) {
            break;
        } else {
            try {
                Thread.sleep(1000);
            } catch (InterruptedException e) {
                e.printStackTrace();
            }
        }
    }
}
Example 16
Source File: BaseSchedulerTest.java    From kylin with Apache License 2.0 5 votes vote down vote up 
protected void waitForJobFinish(String jobId, int maxWaitTime) {
    int error = 0;
    long start = System.currentTimeMillis();
    final int errorLimit = 3;
    while (error < errorLimit && (System.currentTimeMillis() - start < maxWaitTime)) {
        try {
            Thread.sleep(1000);
        } catch (InterruptedException e) {
            e.printStackTrace();
        }

        try {
            AbstractExecutable job = execMgr.getJob(jobId);
            ExecutableState status = job.getStatus();
            if (status == ExecutableState.SUCCEED || status == ExecutableState.ERROR
                    || status == ExecutableState.STOPPED || status == ExecutableState.DISCARDED) {
                break;
            }
        } catch (Exception ex) {
            logger.error("", ex);
            error++;
        }
    }

    if (error >= errorLimit) {
        throw new RuntimeException("too many exceptions");
    }

    if (System.currentTimeMillis() - start >= maxWaitTime) {
        throw new RuntimeException("too long wait time");
    }
}
Example 17
Source File: BaseSchedulerTest.java    From Kylin with Apache License 2.0 5 votes vote down vote up 
protected void waitForJobFinish(String jobId) {
    while (true) {
        AbstractExecutable job = jobService.getJob(jobId);
        final ExecutableState status = job.getStatus();
        if (status == ExecutableState.SUCCEED || status == ExecutableState.ERROR || status == ExecutableState.STOPPED || status == ExecutableState.DISCARDED) {
            break;
        } else {
            try {
                Thread.sleep(5000);
            } catch (InterruptedException e) {
                e.printStackTrace();
            }
        }
    }
}
Example 18
Source File: BuildCubeWithEngine.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
private boolean checkJobState() throws Exception {
    List<String> jobIds = jobService.getAllJobIdsInCache();
    while (true) {
        if (jobIds.size() == 0) {
            return true;
        }
        for (int i = jobIds.size() - 1; i >= 0; i--) {
            String jobId = jobIds.get(i);
            Output job = jobService.getOutputDigest(jobId);
            if (job.getState() == ExecutableState.ERROR) {
                return false;
            } else if (job.getState() == ExecutableState.SUCCEED) {
                String checkActionName = jobCheckActionMap.get(jobId);
                CubeSegment jobSegment = jobSegmentMap.get(jobId);
                if (checkActionName != null) {
                    Method checkAction = this.getClass().getDeclaredMethod(checkActionName, CubeSegment.class);
                    checkAction.invoke(this, jobSegment);
                    jobCheckActionMap.remove(jobId);
                    jobSegmentMap.remove(jobId);
                }
                jobIds.remove(i);
            }
        }
        try {
            Thread.sleep(5000);
        } catch (InterruptedException e) {
            e.printStackTrace();
        }
    }
}
Example 19
Source File: BuildCubeWithEngine.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
protected ExecutableState waitForJob(String jobId) {
    while (true) {
        AbstractExecutable job = jobService.getJob(jobId);
        if (job.getStatus() == ExecutableState.SUCCEED || job.getStatus() == ExecutableState.ERROR) {
            return job.getStatus();
        } else {
            try {
                Thread.sleep(5000);
            } catch (InterruptedException e) {
                e.printStackTrace();
            }
        }
    }
}
Example 20
Source File: CubingJob.java    From kylin with Apache License 2.0 4 votes vote down vote up 
@Override
protected Pair<String, String> formatNotifications(ExecutableContext context, ExecutableState state) {
    CubeInstance cubeInstance = CubeManager.getInstance(context.getConfig())
            .getCube(CubingExecutableUtil.getCubeName(this.getParams()));
    final Output output = getManager().getOutput(getId());
    if (state != ExecutableState.ERROR
            && !cubeInstance.getDescriptor().getStatusNeedNotify().contains(state.toString())) {
        logger.info("state:" + state + " no need to notify users");
        return null;
    }

    if (!MailNotificationUtil.hasMailNotification(state)) {
        logger.info("Cannot find email template for job state: " + state);
        return null;
    }

    Map<String, Object> dataMap = Maps.newHashMap();
    dataMap.put("job_name", getName());
    dataMap.put("env_name", getDeployEnvName());
    dataMap.put("submitter", StringUtil.noBlank(getSubmitter(), "missing submitter"));
    dataMap.put("job_engine", MailNotificationUtil.getLocalHostName());
    dataMap.put("project_name", getProjectName());
    dataMap.put("cube_name", cubeInstance.getName());
    dataMap.put("source_records_count", String.valueOf(findSourceRecordCount()));
    dataMap.put("start_time", new Date(getStartTime()).toString());
    dataMap.put("duration", getDuration() / 60000 + "mins");
    dataMap.put("mr_waiting", getMapReduceWaitTime() / 60000 + "mins");
    dataMap.put("last_update_time", new Date(getLastModified()).toString());

    if (state == ExecutableState.ERROR) {
        AbstractExecutable errorTask = null;
        Output errorOutput = null;
        for (AbstractExecutable task : getTasks()) {
            errorOutput = getManager().getOutput(task.getId());
            if (errorOutput.getState() == ExecutableState.ERROR) {
                errorTask = task;
                break;
            }
        }
        Preconditions.checkNotNull(errorTask,
                "None of the sub tasks of cubing job " + getId() + " is error and this job should become success.");

        dataMap.put("error_step", errorTask.getName());
        if (errorTask instanceof MapReduceExecutable) {
            final String mrJobId = errorOutput.getExtra().get(ExecutableConstants.MR_JOB_ID);
            dataMap.put("mr_job_id", StringUtil.noBlank(mrJobId, "Not initialized"));
        } else {
            dataMap.put("mr_job_id", MailNotificationUtil.NA);
        }
        dataMap.put("error_log",
                Matcher.quoteReplacement(StringUtil.noBlank(output.getVerboseMsg(), "no error message")));
    }

    String content = MailNotificationUtil.getMailContent(state, dataMap);
    String title = MailNotificationUtil.getMailTitle("JOB",
            state.toString(),
            context.getConfig().getClusterName(),
            getDeployEnvName(),
            getProjectName(),
            cubeInstance.getName());
    return Pair.newPair(title, content);
}