org.apache.kylin.job.dao.ExecutableOutputPO Java Examples

public void updateJobOutput(String jobId, ExecutableState newStatus, Map<String, String> info, String output) {
    try {
        final ExecutableOutputPO jobOutput = executableDao.getJobOutput(jobId);
        Preconditions.checkArgument(jobOutput != null, "there is no related output for job id:" + jobId);
        ExecutableState oldStatus = ExecutableState.valueOf(jobOutput.getStatus());
        if (newStatus != null && oldStatus != newStatus) {
            if (!ExecutableState.isValidStateTransfer(oldStatus, newStatus)) {
                throw new IllegalStateTranferException("there is no valid state transfer from:" + oldStatus + " to:" + newStatus);
            }
            jobOutput.setStatus(newStatus.toString());
        }
        if (info != null) {
            jobOutput.setInfo(info);
        }
        if (output != null) {
            jobOutput.setContent(output);
        }
        executableDao.updateJobOutput(jobOutput);
        logger.info("job id:" + jobId + " from " + oldStatus + " to " + newStatus);
    } catch (PersistentException e) {
        logger.error("error change job:" + jobId + " to " + newStatus.toString());
        throw new RuntimeException(e);
    }
}
 

private boolean isJobComplete(ExecutableDao executableDao, ExecutablePO job) {
    String jobId = job.getUuid();
    boolean isComplete = false;
    try {
        ExecutableOutputPO output = executableDao.getJobOutput(jobId);
        String status = output.getStatus();
        String jobType = job.getType();
        if (jobType.equals(CubingJob.class.getName())
                || jobType.equals(CheckpointExecutable.class.getName())) {
            if (StringUtils.equals(status, ExecutableState.SUCCEED.toString())
                    || StringUtils.equals(status, ExecutableState.DISCARDED.toString())) {
                isComplete = true;
            }
        } else if (jobType.equals(CardinalityExecutable.class.getName())) {
            // Ignore state of DefaultChainedExecutable
            isComplete = true;
        }
    } catch (PersistentException e) {
        logger.error("Get job output failed for job uuid: {}", jobId, e);
        isComplete = true; // job output broken --> will be treat as complete
    }

    return isComplete;
}
 

private boolean isJobComplete(ExecutableDao executableDao, ExecutablePO job) {
    String jobId = job.getUuid();
    boolean isComplete = false;
    try {
        ExecutableOutputPO output = executableDao.getJobOutput(jobId);
        String status = output.getStatus();
        String jobType = job.getType();
        if (jobType.equals(CubingJob.class.getName())
                || jobType.equals(CheckpointExecutable.class.getName())) {
            if (StringUtils.equals(status, ExecutableState.SUCCEED.toString())
                    || StringUtils.equals(status, ExecutableState.DISCARDED.toString())) {
                isComplete = true;
            }
        } else if (jobType.equals(CardinalityExecutable.class.getName())) {
            // Ignore state of DefaultChainedExecutable
            isComplete = true;
        }
    } catch (PersistentException e) {
        logger.error("Get job output failed for job uuid: {}", jobId, e);
        isComplete = true; // job output broken --> will be treat as complete
    }

    return isComplete;
}
 

public void forceKillJob(String jobId) {
    try {
        final ExecutableOutputPO jobOutput = executableDao.getJobOutput(jobId);
        List<ExecutablePO> tasks = executableDao.getJob(jobId).getTasks();

        for (ExecutablePO task : tasks) {
            if (executableDao.getJobOutput(task.getId()).getStatus().equals("SUCCEED")) {
                continue;
            } else if (executableDao.getJobOutput(task.getId()).getStatus().equals("RUNNING")) {
                updateJobOutput(task.getId(), ExecutableState.READY, Maps.<String, String> newHashMap(), "");
            }
            break;
        }

        if (!jobOutput.getStatus().equals(ExecutableState.ERROR.toString())) {
            jobOutput.setStatus(ExecutableState.ERROR.toString());
            executableDao.updateJobOutput(jobOutput);
        }
    } catch (PersistentException e) {
        throw new RuntimeException(e);
    }
}
 

public void forceKillJob(String jobId) {
    try {
        final ExecutableOutputPO jobOutput = executableDao.getJobOutput(jobId);
        List<ExecutablePO> tasks = executableDao.getJob(jobId).getTasks();

        for (ExecutablePO task : tasks) {
            if (executableDao.getJobOutput(task.getId()).getStatus().equals("SUCCEED")) {
                continue;
            } else if (executableDao.getJobOutput(task.getId()).getStatus().equals("RUNNING")) {
                updateJobOutput(task.getId(), ExecutableState.READY, Maps.<String, String> newHashMap(), "");
            }
            break;
        }

        if (!jobOutput.getStatus().equals(ExecutableState.ERROR.toString())) {
            jobOutput.setStatus(ExecutableState.ERROR.toString());
            executableDao.updateJobOutput(jobOutput);
        }
    } catch (PersistentException e) {
        throw new RuntimeException(e);
    }
}
 

private DefaultOutput parseOutput(ExecutableOutputPO jobOutput) {
    final DefaultOutput result = new DefaultOutput();
    result.setExtra(jobOutput.getInfo());
    result.setState(ExecutableState.valueOf(jobOutput.getStatus()));
    result.setVerboseMsg(jobOutput.getContent());
    result.setLastModified(jobOutput.getLastModified());
    return result;
}
 

public Output getOutput(String uuid) {
    try {
        uuid = uuid.replaceAll("[./]", "");
        final ExecutableOutputPO jobOutput = executableDao.getJobOutput(uuid);
        Preconditions.checkArgument(jobOutput != null, "there is no related output for job id:" + uuid);
        return parseOutput(jobOutput);
    } catch (PersistentException e) {
        logger.error("fail to get job output:" + uuid, e);
        throw new RuntimeException(e);
    }
}
 

public Map<String, Output> getAllOutputs() {
    try {
        final List<ExecutableOutputPO> jobOutputs = executableDao.getJobOutputs();
        HashMap<String, Output> result = Maps.newHashMap();
        for (ExecutableOutputPO jobOutput : jobOutputs) {
            result.put(jobOutput.getId(), parseOutput(jobOutput));
        }
        return result;
    } catch (PersistentException e) {
        logger.error("fail to get all job output:", e);
        throw new RuntimeException(e);
    }
}
 

public Map<String, Output> getAllOutputs(long timeStartInMillis, long timeEndInMillis) {
    try {
        final List<ExecutableOutputPO> jobOutputs = executableDao.getJobOutputs(timeStartInMillis, timeEndInMillis);
        HashMap<String, Output> result = Maps.newHashMap();
        for (ExecutableOutputPO jobOutput : jobOutputs) {
            result.put(jobOutput.getId(), parseOutput(jobOutput));
        }
        return result;
    } catch (PersistentException e) {
        logger.error("fail to get all job output:", e);
        throw new RuntimeException(e);
    }
}
 

public Map<String, ExecutableOutputPO> getAllOutputDigests(long timeStartInMillis, long timeEndInMillis) {
    final List<ExecutableOutputPO> jobOutputs = executableDao.getJobOutputDigests(timeStartInMillis,
            timeEndInMillis);
    HashMap<String, ExecutableOutputPO> result = Maps.newHashMap();
    for (ExecutableOutputPO jobOutput : jobOutputs) {
        result.put(jobOutput.getId(), jobOutput);
    }
    return result;
}
 

public void updateAllRunningJobsToError() {
    try {
        final List<ExecutableOutputPO> jobOutputs = executableDao.getJobOutputs();
        for (ExecutableOutputPO executableOutputPO : jobOutputs) {
            if (executableOutputPO.getStatus().equalsIgnoreCase(ExecutableState.RUNNING.toString())) {
                executableOutputPO.setStatus(ExecutableState.ERROR.toString());
                executableDao.updateJobOutput(executableOutputPO);
            }
        }
    } catch (PersistentException e) {
        logger.error("error reset job status from RUNNING to ERROR", e);
        throw new RuntimeException(e);
    }
}
 

public void resumeAllRunningJobs() {
    try {
        final List<ExecutableOutputPO> jobOutputs = executableDao.getJobOutputs();
        for (ExecutableOutputPO executableOutputPO : jobOutputs) {
            if (executableOutputPO.getStatus().equalsIgnoreCase(ExecutableState.RUNNING.toString())) {
                executableOutputPO.setStatus(ExecutableState.READY.toString());
                executableDao.updateJobOutput(executableOutputPO);
            }
        }
    } catch (PersistentException e) {
        logger.error("error reset job status from RUNNING to READY", e);
        throw new RuntimeException(e);
    }
}
 

public ExecutableOutputPO getJobOutput(String jobId) {
    try {
        return executableDao.getJobOutput(jobId);
    } catch (PersistentException e) {
        logger.error("Can't get output of Job " + jobId);
        throw new RuntimeException(e);
    }
}
 

public void updateJobOutput(String jobId, ExecutableState newStatus, Map<String, String> info, String output) {
    // when 
    if (Thread.currentThread().isInterrupted()) {
        throw new RuntimeException("Current thread is interruptted, aborting");
    }

    try {
        final ExecutableOutputPO jobOutput = executableDao.getJobOutput(jobId);
        Preconditions.checkArgument(jobOutput != null, "there is no related output for job id:" + jobId);
        ExecutableState oldStatus = ExecutableState.valueOf(jobOutput.getStatus());
        if (newStatus != null && oldStatus != newStatus) {
            if (!ExecutableState.isValidStateTransfer(oldStatus, newStatus)) {
                throw new IllegalStateTranferException("there is no valid state transfer from:" + oldStatus + " to:"
                        + newStatus + ", job id: " + jobId);
            }
            jobOutput.setStatus(newStatus.toString());
        }
        if (info != null) {
            jobOutput.setInfo(info);
        }
        if (output != null) {
            if (output.length() > config.getJobOutputMaxSize()) {
                output = output.substring(0, config.getJobOutputMaxSize());
            }
            jobOutput.setContent(output);
        }
        executableDao.updateJobOutput(jobOutput);
        logger.info("job id:" + jobId + " from " + oldStatus + " to " + newStatus);
    } catch (PersistentException e) {
        logger.error("error change job:" + jobId + " to " + newStatus);
        throw new RuntimeException(e);
    }
}
 

public void resetJobOutput(String jobId, ExecutableState state, String output) {
    try {
        final ExecutableOutputPO jobOutput = executableDao.getJobOutput(jobId);
        jobOutput.setStatus(state.toString());
        if (output != null) {
            jobOutput.setContent(output);
        }
        executableDao.updateJobOutput(jobOutput);
    } catch (PersistentException e) {
        throw new RuntimeException(e);
    }
}
 

public List<JobSearchResult> innerSearchCubingJobsV2(final String cubeName, final String jobName,
        final String projectName, final List<JobStatusEnum> statusList, final JobTimeFilterEnum timeFilter) {
    if (null == projectName) {
        aclEvaluate.checkIsGlobalAdmin();
    } else {
        aclEvaluate.checkProjectOperationPermission(projectName);
    }
    // prepare time range
    Calendar calendar = Calendar.getInstance(TimeZone.getDefault(), Locale.ROOT);
    calendar.setTime(new Date());
    long timeStartInMillis = getTimeStartInMillis(calendar, timeFilter);
    long timeEndInMillis = Long.MAX_VALUE;
    Set<ExecutableState> states = convertStatusEnumToStates(statusList);
    final Map<String, ExecutableOutputPO> allOutputDigests = getExecutableManager()
            .getAllOutputDigests(timeStartInMillis, timeEndInMillis);
    return Lists
            .newArrayList(FluentIterable
                    .from(innerSearchCubingJobsV2(cubeName, jobName, states, timeStartInMillis, timeEndInMillis,
                            allOutputDigests, false, projectName))
                    .transform(new Function<CubingJob, JobSearchResult>() {
                        @Override
                        public JobSearchResult apply(CubingJob cubingJob) {
                            return JobInfoConverter.parseToJobSearchResult(cubingJob, allOutputDigests);
                        }
                    }).filter(new Predicate<JobSearchResult>() {
                        @Override
                        public boolean apply(@Nullable JobSearchResult input) {
                            return input != null;
                        }
                    }));
}
 

public List<JobSearchResult> innerSearchCheckpointJobsV2(final String cubeName, final String jobName,
        final String projectName, final List<JobStatusEnum> statusList, final JobTimeFilterEnum timeFilter) {
    if (null == projectName) {
        aclEvaluate.checkIsGlobalAdmin();
    } else {
        aclEvaluate.checkProjectOperationPermission(projectName);
    }
    // prepare time range
    Calendar calendar = Calendar.getInstance(TimeZone.getDefault(), Locale.ROOT);
    calendar.setTime(new Date());
    long timeStartInMillis = getTimeStartInMillis(calendar, timeFilter);
    long timeEndInMillis = Long.MAX_VALUE;
    Set<ExecutableState> states = convertStatusEnumToStates(statusList);
    final Map<String, ExecutableOutputPO> allOutputDigests = getExecutableManager()
            .getAllOutputDigests(timeStartInMillis, timeEndInMillis);
    return Lists.newArrayList(FluentIterable
            .from(innerSearchCheckpointJobsV2(cubeName, jobName, states, timeStartInMillis, timeEndInMillis,
                    allOutputDigests, false, projectName))
            .transform(new Function<CheckpointExecutable, JobSearchResult>() {
                @Override
                public JobSearchResult apply(CheckpointExecutable checkpointExecutable) {
                    return JobInfoConverter.parseToJobSearchResult(checkpointExecutable, allOutputDigests);
                }
            }).filter(new Predicate<JobSearchResult>() {
                @Override
                public boolean apply(@Nullable JobSearchResult input) {
                    return input != null;
                }
            }));
}
 

private void addJobOutput(AbstractExecutable executable) throws PersistentException {
    ExecutableOutputPO executableOutputPO = new ExecutableOutputPO();
    executableOutputPO.setUuid(executable.getId());
    executableDao.addJobOutput(executableOutputPO);
    if (executable instanceof DefaultChainedExecutable) {
        for (AbstractExecutable subTask: ((DefaultChainedExecutable) executable).getTasks()) {
            addJobOutput(subTask);
        }
    }
}
 

public Output getOutput(String uuid) {
    try {
        final ExecutableOutputPO jobOutput = executableDao.getJobOutput(uuid);
        Preconditions.checkArgument(jobOutput != null, "there is no related output for job id:" + uuid);
        final DefaultOutput result = new DefaultOutput();
        result.setExtra(jobOutput.getInfo());
        result.setState(ExecutableState.valueOf(jobOutput.getStatus()));
        result.setVerboseMsg(jobOutput.getContent());
        result.setLastModified(jobOutput.getLastModified());
        return result;
    } catch (PersistentException e) {
        logger.error("fail to get job output:" + uuid, e);
        throw new RuntimeException(e);
    }
}
 

public void updateAllRunningJobsToError() {
    try {
        final List<ExecutableOutputPO> jobOutputs = executableDao.getJobOutputs();
        for (ExecutableOutputPO executableOutputPO : jobOutputs) {
            if (executableOutputPO.getStatus().equalsIgnoreCase(ExecutableState.RUNNING.toString())) {
                executableOutputPO.setStatus(ExecutableState.ERROR.toString());
                executableDao.updateJobOutput(executableOutputPO);
            }
        }
    } catch (PersistentException e) {
        logger.error("error reset job status from RUNNING to ERROR", e);
        throw new RuntimeException(e);
    }
}
 

public void resetJobOutput(String jobId, ExecutableState state, String output) {
    try {
        final ExecutableOutputPO jobOutput = executableDao.getJobOutput(jobId);
        jobOutput.setStatus(state.toString());
        if (output != null) {
            jobOutput.setContent(output);
        }
        executableDao.updateJobOutput(jobOutput);
    } catch (PersistentException e) {
        throw new RuntimeException(e);
    }
}
 

public void addJobInfo(String id, Map<String, String> info) {
    if (info == null) {
        return;
    }
    try {
        ExecutableOutputPO output = executableDao.getJobOutput(id);
        Preconditions.checkArgument(output != null, "there is no related output for job id:" + id);
        output.getInfo().putAll(info);
        executableDao.updateJobOutput(output);
    } catch (PersistentException e) {
        logger.error("error update job info, id:" + id + "  info:" + info.toString());
        throw new RuntimeException(e);
    }
}
 

public void resumeAllRunningJobs() {
    try {
        final List<ExecutableOutputPO> jobOutputs = executableDao.getJobOutputs();
        for (ExecutableOutputPO executableOutputPO : jobOutputs) {
            if (executableOutputPO.getStatus().equalsIgnoreCase(ExecutableState.RUNNING.toString())) {
                executableOutputPO.setStatus(ExecutableState.READY.toString());
                executableDao.updateJobOutput(executableOutputPO);
            }
        }
    } catch (PersistentException e) {
        logger.error("error reset job status from RUNNING to READY", e);
        throw new RuntimeException(e);
    }
}
 

public static JobSearchResult parseToJobSearchResult(DefaultChainedExecutable job, Map<String, ExecutableOutputPO> outputs) {
    if (job == null) {
        logger.warn("job is null.");
        return null;
    }

    ExecutableOutputPO output = outputs.get(job.getId());
    if (output == null) {
        logger.warn("job output is null.");
        return null;
    }

    final JobSearchResult result = new JobSearchResult();

    String cubeName = CubingExecutableUtil.getCubeName(job.getParams());

    if (cubeName == null) {
        cubeName = job.getParam("model_name");
    } else {
        CubeInstance cube = CubeManager.getInstance(KylinConfig.getInstanceFromEnv()).getCube(cubeName);
        if (cube != null) {
            cubeName = cube.getDisplayName();
        }
    }
    result.setCubeName(cubeName);
    result.setId(job.getId());
    result.setJobName(job.getName());
    result.setLastModified(output.getLastModified());
    result.setJobStatus(JobInfoConverter.parseToJobStatus(job.getStatus()));
    return result;
}
 

private void addJobOutput(AbstractExecutable executable) throws PersistentException {
    ExecutableOutputPO executableOutputPO = new ExecutableOutputPO();
    executableOutputPO.setUuid(executable.getId());
    executableDao.addJobOutput(executableOutputPO);
    if (executable instanceof DefaultChainedExecutable) {
        for (AbstractExecutable subTask : ((DefaultChainedExecutable) executable).getTasks()) {
            addJobOutput(subTask);
        }
    }
}
 

public Output getOutput(String uuid) {
    try {
        uuid = uuid.replaceAll("[./]", "");
        final ExecutableOutputPO jobOutput = executableDao.getJobOutput(uuid);
        Preconditions.checkArgument(jobOutput != null, "there is no related output for job id:" + uuid);
        return parseOutput(jobOutput);
    } catch (PersistentException e) {
        logger.error("fail to get job output:" + uuid, e);
        throw new RuntimeException(e);
    }
}
 

private DefaultOutput parseOutput(ExecutableOutputPO jobOutput) {
    final DefaultOutput result = new DefaultOutput();
    result.setExtra(jobOutput.getInfo());
    result.setState(ExecutableState.valueOf(jobOutput.getStatus()));
    result.setVerboseMsg(jobOutput.getContent());
    result.setLastModified(jobOutput.getLastModified());
    return result;
}
 

public Map<String, Output> getAllOutputs() {
    try {
        final List<ExecutableOutputPO> jobOutputs = executableDao.getJobOutputs();
        HashMap<String, Output> result = Maps.newHashMap();
        for (ExecutableOutputPO jobOutput : jobOutputs) {
            result.put(jobOutput.getId(), parseOutput(jobOutput));
        }
        return result;
    } catch (PersistentException e) {
        logger.error("fail to get all job output:", e);
        throw new RuntimeException(e);
    }
}
 

public Map<String, Output> getAllOutputs(long timeStartInMillis, long timeEndInMillis) {
    try {
        final List<ExecutableOutputPO> jobOutputs = executableDao.getJobOutputs(timeStartInMillis, timeEndInMillis);
        HashMap<String, Output> result = Maps.newHashMap();
        for (ExecutableOutputPO jobOutput : jobOutputs) {
            result.put(jobOutput.getId(), parseOutput(jobOutput));
        }
        return result;
    } catch (PersistentException e) {
        logger.error("fail to get all job output:", e);
        throw new RuntimeException(e);
    }
}
 

public Map<String, ExecutableOutputPO> getAllOutputDigests(long timeStartInMillis, long timeEndInMillis) {
    final List<ExecutableOutputPO> jobOutputs = executableDao.getJobOutputDigests(timeStartInMillis,
            timeEndInMillis);
    HashMap<String, ExecutableOutputPO> result = Maps.newHashMap();
    for (ExecutableOutputPO jobOutput : jobOutputs) {
        result.put(jobOutput.getId(), jobOutput);
    }
    return result;
}