org.apache.kylin.job.exception.PersistentException Java Examples

public void updateJobOutput(String jobId, ExecutableState newStatus, Map<String, String> info, String output) {
    try {
        final ExecutableOutputPO jobOutput = executableDao.getJobOutput(jobId);
        Preconditions.checkArgument(jobOutput != null, "there is no related output for job id:" + jobId);
        ExecutableState oldStatus = ExecutableState.valueOf(jobOutput.getStatus());
        if (newStatus != null && oldStatus != newStatus) {
            if (!ExecutableState.isValidStateTransfer(oldStatus, newStatus)) {
                throw new IllegalStateTranferException("there is no valid state transfer from:" + oldStatus + " to:" + newStatus);
            }
            jobOutput.setStatus(newStatus.toString());
        }
        if (info != null) {
            jobOutput.setInfo(info);
        }
        if (output != null) {
            jobOutput.setContent(output);
        }
        executableDao.updateJobOutput(jobOutput);
        logger.info("job id:" + jobId + " from " + oldStatus + " to " + newStatus);
    } catch (PersistentException e) {
        logger.error("error change job:" + jobId + " to " + newStatus.toString());
        throw new RuntimeException(e);
    }
}
 

public void forceKillJob(String jobId) {
    try {
        final ExecutableOutputPO jobOutput = executableDao.getJobOutput(jobId);
        List<ExecutablePO> tasks = executableDao.getJob(jobId).getTasks();

        for (ExecutablePO task : tasks) {
            if (executableDao.getJobOutput(task.getId()).getStatus().equals("SUCCEED")) {
                continue;
            } else if (executableDao.getJobOutput(task.getId()).getStatus().equals("RUNNING")) {
                updateJobOutput(task.getId(), ExecutableState.READY, Maps.<String, String> newHashMap(), "");
            }
            break;
        }

        if (!jobOutput.getStatus().equals(ExecutableState.ERROR.toString())) {
            jobOutput.setStatus(ExecutableState.ERROR.toString());
            executableDao.updateJobOutput(jobOutput);
        }
    } catch (PersistentException e) {
        throw new RuntimeException(e);
    }
}
 

public List<String> getJobIds() throws PersistentException {
    try {
        ArrayList<String> resources = store.listResources(JOB_PATH_ROOT);
        if (resources == null) {
            return Collections.emptyList();
        }
        ArrayList<String> result = Lists.newArrayListWithExpectedSize(resources.size());
        for (String path : resources) {
            result.add(path.substring(path.lastIndexOf("/") + 1));
        }
        return result;
    } catch (IOException e) {
        logger.error("error get all Jobs:", e);
        throw new PersistentException(e);
    }
}
 

public void forceKillJob(String jobId) {
    try {
        final ExecutableOutputPO jobOutput = executableDao.getJobOutput(jobId);
        List<ExecutablePO> tasks = executableDao.getJob(jobId).getTasks();

        for (ExecutablePO task : tasks) {
            if (executableDao.getJobOutput(task.getId()).getStatus().equals("SUCCEED")) {
                continue;
            } else if (executableDao.getJobOutput(task.getId()).getStatus().equals("RUNNING")) {
                updateJobOutput(task.getId(), ExecutableState.READY, Maps.<String, String> newHashMap(), "");
            }
            break;
        }

        if (!jobOutput.getStatus().equals(ExecutableState.ERROR.toString())) {
            jobOutput.setStatus(ExecutableState.ERROR.toString());
            executableDao.updateJobOutput(jobOutput);
        }
    } catch (PersistentException e) {
        throw new RuntimeException(e);
    }
}
 

public void updateCheckpointJob(String jobId, List<AbstractExecutable> subTasksForCheck) {
    try {
        jobId = jobId.replaceAll("[./]", "");
        final ExecutablePO job = executableDao.getJob(jobId);
        Preconditions.checkArgument(job != null, "there is no related job for job id:" + jobId);

        List<ExecutablePO> tasksForCheck = Lists.newArrayListWithExpectedSize(subTasksForCheck.size());
        for (AbstractExecutable taskForCheck : subTasksForCheck) {
            tasksForCheck.add(parse(taskForCheck));
        }
        job.setTasksForCheck(tasksForCheck);
        executableDao.updateJob(job);
    } catch (PersistentException e) {
        logger.error("fail to update checkpoint job:" + jobId, e);
        throw new RuntimeException(e);
    }
}
 

private boolean isJobComplete(ExecutableDao executableDao, ExecutablePO job) {
    String jobId = job.getUuid();
    boolean isComplete = false;
    try {
        ExecutableOutputPO output = executableDao.getJobOutput(jobId);
        String status = output.getStatus();
        String jobType = job.getType();
        if (jobType.equals(CubingJob.class.getName())
                || jobType.equals(CheckpointExecutable.class.getName())) {
            if (StringUtils.equals(status, ExecutableState.SUCCEED.toString())
                    || StringUtils.equals(status, ExecutableState.DISCARDED.toString())) {
                isComplete = true;
            }
        } else if (jobType.equals(CardinalityExecutable.class.getName())) {
            // Ignore state of DefaultChainedExecutable
            isComplete = true;
        }
    } catch (PersistentException e) {
        logger.error("Get job output failed for job uuid: {}", jobId, e);
        isComplete = true; // job output broken --> will be treat as complete
    }

    return isComplete;
}
 

public List<ExecutablePO> getJobs() throws PersistentException {
    try {
        ArrayList<String> resources = store.listResources(JOB_PATH_ROOT);
        if (resources == null) {
            return Collections.emptyList();
        }
        ArrayList<ExecutablePO> result = new ArrayList<ExecutablePO>(resources.size());
        for (String path : resources) {
            result.add(readJobResource(path));
        }
        return result;
    } catch (IOException e) {
        logger.error("error get all Jobs:", e);
        throw new PersistentException(e);
    }
}
 

public static boolean isMetaDataPersistException(Exception e, final int maxDepth) {
    if (e instanceof PersistentException) {
        return true;
    }

    Throwable t = e.getCause();
    int depth = 0;
    while (t != null && depth < maxDepth) {
        depth++;
        if (t instanceof PersistentException) {
            return true;
        }
        t = t.getCause();
    }
    return false;
}
 

public void syncDigestsOfJob(String uuid) throws PersistentException {
    ExecutablePO job = getJob(uuid);
    ExecutablePO jobDigest = getJobDigest(uuid);

    if (job == null && jobDigest != null) {
        executableDigestMap.remove(uuid);
    } else if (job != null && jobDigest == null) {
        executableDigestMap.put(uuid, job);
    }

    ExecutableOutputPO jobOutput = getJobOutput(uuid);
    ExecutableOutputPO jobOutputDigest = getJobOutputDigest(uuid);

    if (jobOutput == null && jobOutputDigest != null) {
        executableOutputDigestMap.remove(uuid);
    } else if (jobOutput != null && jobOutputDigest == null) {
        executableOutputDigestMap.put(uuid, jobOutput);
    }
}
 

private boolean isJobComplete(ExecutableDao executableDao, ExecutablePO job) {
    String jobId = job.getUuid();
    boolean isComplete = false;
    try {
        ExecutableOutputPO output = executableDao.getJobOutput(jobId);
        String status = output.getStatus();
        String jobType = job.getType();
        if (jobType.equals(CubingJob.class.getName())
                || jobType.equals(CheckpointExecutable.class.getName())) {
            if (StringUtils.equals(status, ExecutableState.SUCCEED.toString())
                    || StringUtils.equals(status, ExecutableState.DISCARDED.toString())) {
                isComplete = true;
            }
        } else if (jobType.equals(CardinalityExecutable.class.getName())) {
            // Ignore state of DefaultChainedExecutable
            isComplete = true;
        }
    } catch (PersistentException e) {
        logger.error("Get job output failed for job uuid: {}", jobId, e);
        isComplete = true; // job output broken --> will be treat as complete
    }

    return isComplete;
}
 

public ExecutableOutputPO getJobOutput(String uuid) throws PersistentException {
    ExecutableOutputPO result = null;
    try {
        result = readJobOutputResource(pathOfJobOutput(uuid));
        if (result == null) {
            result = new ExecutableOutputPO();
            result.setUuid(uuid);
            return result;
        }
        return result;
    } catch (IOException e) {
        logger.error("error get job output id:" + uuid, e);
        if (e.getCause() instanceof FileNotFoundException) {
            result = new ExecutableOutputPO();
            result.setUuid(uuid);
            result.setStatus(ExecutableState.SUCCEED.name());
            return result;
        } else {
            throw new PersistentException(e);
        }
    }
}
 

public List<ExecutableOutputPO> getJobOutputs() throws PersistentException {
    try {
        ArrayList<String> resources = store.listResources(JOB_OUTPUT_ROOT);
        if (resources == null) {
            return Collections.emptyList();
        }
        ArrayList<ExecutableOutputPO> result = new ArrayList<ExecutableOutputPO>(resources.size());
        for (String path : resources) {
            result.add(readJobOutputResource(path));
        }
        return result;
    } catch (IOException e) {
        logger.error("error get all Jobs:", e);
        throw new PersistentException(e);
    }
}
 

public static boolean isMetaDataPersistException(Exception e, final int maxDepth) {
    if (e instanceof PersistentException) {
        return true;
    }

    Throwable t = e.getCause();
    int depth = 0;
    while (t != null && depth < maxDepth) {
        depth++;
        if (t instanceof PersistentException) {
            return true;
        }
        t = t.getCause();
    }
    return false;
}
 

public void syncDigestsOfJob(String uuid) throws PersistentException {
    ExecutablePO job = getJob(uuid);
    ExecutablePO jobDigest = getJobDigest(uuid);

    if (job == null && jobDigest != null) {
        executableDigestMap.remove(uuid);
    } else if (job != null && jobDigest == null) {
        executableDigestMap.put(uuid, job);
    }

    ExecutableOutputPO jobOutput = getJobOutput(uuid);
    ExecutableOutputPO jobOutputDigest = getJobOutputDigest(uuid);

    if (jobOutput == null && jobOutputDigest != null) {
        executableOutputDigestMap.remove(uuid);
    } else if (jobOutput != null && jobOutputDigest == null) {
        executableOutputDigestMap.put(uuid, jobOutput);
    }
}
 

@Test
public void testBasics() throws IOException, PersistentException {
    CubeDesc cubeDesc = cubeDescManager.getCubeDesc("test_kylin_cube_with_slr_left_join_desc");
    CubeInstance cube = cubeManager.createCube(CUBE_NAME, "DEFAULT", cubeDesc, "test");
    assertNotNull(cube);

    JobListRequest jobRequest = new JobListRequest();
    Assert.assertNotNull(jobSchedulerController.list(jobRequest));

    JobBuildRequest jobBuildRequest = new JobBuildRequest();
    jobBuildRequest.setBuildType("BUILD");
    jobBuildRequest.setStartTime(0L);
    jobBuildRequest.setEndTime(new Date().getTime());
    JobInstance job = cubeController.rebuild(CUBE_NAME, jobBuildRequest);

    Assert.assertNotNull(jobSchedulerController.get(job.getId()));
    executableDAO.deleteJob(job.getId());
    if (cubeManager.getCube(CUBE_NAME) != null) {
        cubeManager.dropCube(CUBE_NAME, false);
    }

    // jobSchedulerController.cancel(job.getId());
}
 

public List<String> getJobIds() throws PersistentException {
    try {
        NavigableSet<String> resources = store.listResources(ResourceStore.EXECUTE_RESOURCE_ROOT);
        if (resources == null) {
            return Collections.emptyList();
        }
        ArrayList<String> result = Lists.newArrayListWithExpectedSize(resources.size());
        for (String path : resources) {
            result.add(path.substring(path.lastIndexOf("/") + 1));
        }
        return result;
    } catch (IOException e) {
        logger.error("error get all Jobs:", e);
        throw new PersistentException(e);
    }
}
 

public List<String> getJobIds() throws PersistentException {
    try {
        NavigableSet<String> resources = store.listResources(ResourceStore.EXECUTE_RESOURCE_ROOT);
        if (resources == null) {
            return Collections.emptyList();
        }
        ArrayList<String> result = Lists.newArrayListWithExpectedSize(resources.size());
        for (String path : resources) {
            result.add(path.substring(path.lastIndexOf("/") + 1));
        }
        return result;
    } catch (IOException e) {
        logger.error("error get all Jobs:", e);
        throw new PersistentException(e);
    }
}
 

public ExecutableOutputPO getJobOutput(String uuid) throws PersistentException {
    ExecutableOutputPO result = null;
    try {
        result = readJobOutputResource(pathOfJobOutput(uuid));
        if (result == null) {
            result = new ExecutableOutputPO();
            result.setUuid(uuid);
            return result;
        }
        return result;
    } catch (IOException e) {
        logger.error("error get job output id:" + uuid, e);
        if (e.getCause() instanceof FileNotFoundException) {
            result = new ExecutableOutputPO();
            result.setUuid(uuid);
            result.setStatus(ExecutableState.SUCCEED.name());
            return result;
        } else {
            throw new PersistentException(e);
        }
    }
}
 

public AbstractExecutable getJob(String uuid) {
    try {
        return parseTo(executableDao.getJob(uuid));
    } catch (PersistentException e) {
        logger.error("fail to get job:" + uuid, e);
        throw new RuntimeException(e);
    }
}
 

public Map<String, Output> getAllOutputs() {
    try {
        final List<ExecutableOutputPO> jobOutputs = executableDao.getJobOutputs();
        HashMap<String, Output> result = Maps.newHashMap();
        for (ExecutableOutputPO jobOutput : jobOutputs) {
            result.put(jobOutput.getId(), parseOutput(jobOutput));
        }
        return result;
    } catch (PersistentException e) {
        logger.error("fail to get all job output:", e);
        throw new RuntimeException(e);
    }
}
 

public Output getOutput(String uuid) {
    try {
        uuid = uuid.replaceAll("[./]", "");
        final ExecutableOutputPO jobOutput = executableDao.getJobOutput(uuid);
        Preconditions.checkArgument(jobOutput != null, "there is no related output for job id:" + uuid);
        return parseOutput(jobOutput);
    } catch (PersistentException e) {
        logger.error("fail to get job output:" + uuid, e);
        throw new RuntimeException(e);
    }
}
 

private void addJobOutput(AbstractExecutable executable) throws PersistentException {
    ExecutableOutputPO executableOutputPO = new ExecutableOutputPO();
    executableOutputPO.setUuid(executable.getId());
    executableDao.addJobOutput(executableOutputPO);
    if (executable instanceof DefaultChainedExecutable) {
        for (AbstractExecutable subTask: ((DefaultChainedExecutable) executable).getTasks()) {
            addJobOutput(subTask);
        }
    }
}
 

public ExecutableOutputPO getJobOutput(String jobId) {
    try {
        return executableDao.getJobOutput(jobId);
    } catch (PersistentException e) {
        logger.error("Can't get output of Job " + jobId);
        throw new RuntimeException(e);
    }
}
 

public AbstractExecutable getJob(String uuid) {
    try {
        uuid = uuid.replaceAll("[./]", "");
        return parseTo(executableDao.getJob(uuid));
    } catch (PersistentException e) {
        logger.error("fail to get job:" + uuid, e);
        throw new RuntimeException(e);
    }
}
 

public void deleteJob(String jobId) {
    try {
        jobId = jobId.replaceAll("[./]", "");
        executableDao.deleteJob(jobId);
    } catch (PersistentException e) {
        logger.error("fail to delete job:" + jobId, e);
        throw new RuntimeException(e);
    }
}
 

private void addJobOutput(AbstractExecutable executable) throws PersistentException {
    ExecutableOutputPO executableOutputPO = new ExecutableOutputPO();
    executableOutputPO.setUuid(executable.getId());
    executableDao.addJobOutput(executableOutputPO);
    if (executable instanceof DefaultChainedExecutable) {
        for (AbstractExecutable subTask : ((DefaultChainedExecutable) executable).getTasks()) {
            addJobOutput(subTask);
        }
    }
}
 

public void addJob(AbstractExecutable executable) {
    try {
        executable.initConfig(config);
        if (executableDao.getJob(executable.getId()) != null) {
            throw new IllegalArgumentException("job id:" + executable.getId() + " already exists");
        }
        addJobOutput(executable);
        executableDao.addJob(parse(executable));
    } catch (PersistentException e) {
        logger.error("fail to submit job:" + executable.getId(), e);
        throw new RuntimeException(e);
    }
}
 

public void deleteJobOutput(String uuid) throws PersistentException {
    try {
        store.deleteResource(pathOfJobOutput(uuid));
        if (!isTaskExecutableOutput(uuid))
            executableOutputDigestMap.remove(uuid);
    } catch (IOException e) {
        logger.error("error delete job:" + uuid, e);
        throw new PersistentException(e);
    }
}
 

public void addJobOutput(ExecutableOutputPO output) throws PersistentException {
    try {
        output.setLastModified(0);
        writeJobOutputResource(pathOfJobOutput(output.getUuid()), output);
        if (!isTaskExecutableOutput(output.getUuid()))
            executableOutputDigestMap.put(output.getUuid(), output);
    } catch (IOException e) {
        logger.error("error update job output id:" + output.getUuid(), e);
        throw new PersistentException(e);
    }
}
 

public void deleteJob(String uuid) throws PersistentException {
    try {
        ExecutablePO executablePO = getJob(uuid);
        store.deleteResource(pathOfJob(uuid));
        executableDigestMap.remove(uuid);
        removeJobOutput(executablePO);
    } catch (IOException e) {
        logger.error("error delete job:" + uuid, e);
        throw new PersistentException(e);
    }
}