Java Code Examples for org.apache.kylin.job.execution.Output#getState()

The following examples show how to use org.apache.kylin.job.execution.Output#getState() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: DistributedScheduler.java From kylin-on-parquet-v2 with Apache License 2.0

6 votes

private void resumeAllRunningJobs() {
    for (final String id : executableManager.getAllJobIds()) {
        final Output output = executableManager.getOutput(id);
        AbstractExecutable executable = executableManager.getJob(id);
        if (output.getState() == ExecutableState.RUNNING && executable instanceof DefaultChainedExecutable) {
            try {
                if (!jobLock.isLocked(getLockPath(executable.getId()))) {
                    executableManager.resumeRunningJobForce(executable.getId());
                    fetcherPool.schedule(fetcher, 0, TimeUnit.SECONDS);
                }
            } catch (Exception e) {
                logger.error("resume the job " + id + " fail in server: " + serverName, e);
            }
        }
    }
}

Example 2

Source File: FetcherRunner.java From kylin-on-parquet-v2 with Apache License 2.0

6 votes

protected void jobStateCount(String id) {
    final Output outputDigest = getExecutableManager().getOutputDigest(id);
    // logger.debug("Job id:" + id + " not runnable");
    if (outputDigest.getState() == ExecutableState.SUCCEED) {
        succeedJobs.add(id);
        nSUCCEED++;
    } else if (outputDigest.getState() == ExecutableState.ERROR) {
        nError++;
    } else if (outputDigest.getState() == ExecutableState.DISCARDED) {
        nDiscarded++;
    } else if (outputDigest.getState() == ExecutableState.STOPPED) {
        nStopped++;
    } else {
        if (fetchFailed) {
            getExecutableManager().forceKillJob(id);
            nError++;
        } else {
            nOthers++;
        }
    }
}

Example 3

Source File: DistributedScheduler.java From kylin with Apache License 2.0

6 votes

private void resumeAllRunningJobs() {
    for (final String id : executableManager.getAllJobIds()) {
        final Output output = executableManager.getOutput(id);
        AbstractExecutable executable = executableManager.getJob(id);
        if (output.getState() == ExecutableState.RUNNING && executable instanceof DefaultChainedExecutable) {
            try {
                if (!jobLock.isLocked(getLockPath(executable.getId()))) {
                    executableManager.resumeRunningJobForce(executable.getId());
                    fetcherPool.schedule(fetcher, 0, TimeUnit.SECONDS);
                }
            } catch (Exception e) {
                logger.error("resume the job " + id + " fail in server: " + serverName, e);
            }
        }
    }
}

Example 4

Source File: FetcherRunner.java From kylin with Apache License 2.0

6 votes

protected void jobStateCount(String id) {
    final Output outputDigest = getExecutableManager().getOutputDigest(id);
    // logger.debug("Job id:" + id + " not runnable");
    if (outputDigest.getState() == ExecutableState.SUCCEED) {
        succeedJobs.add(id);
        nSUCCEED++;
    } else if (outputDigest.getState() == ExecutableState.ERROR) {
        nError++;
    } else if (outputDigest.getState() == ExecutableState.DISCARDED) {
        nDiscarded++;
    } else if (outputDigest.getState() == ExecutableState.STOPPED) {
        nStopped++;
    } else {
        if (fetchFailed) {
            getExecutableManager().forceKillJob(id);
            nError++;
        } else {
            nOthers++;
        }
    }
}

Example 5

Source File: BuildCubeWithEngine.java From kylin-on-parquet-v2 with Apache License 2.0

5 votes

private boolean checkJobState() throws Exception {
    List<String> jobIds = jobService.getAllJobIdsInCache();
    while (true) {
        if (jobIds.size() == 0) {
            return true;
        }
        for (int i = jobIds.size() - 1; i >= 0; i--) {
            String jobId = jobIds.get(i);
            Output job = jobService.getOutputDigest(jobId);
            if (job.getState() == ExecutableState.ERROR) {
                return false;
            } else if (job.getState() == ExecutableState.SUCCEED) {
                String checkActionName = jobCheckActionMap.get(jobId);
                CubeSegment jobSegment = jobSegmentMap.get(jobId);
                if (checkActionName != null) {
                    Method checkAction = this.getClass().getDeclaredMethod(checkActionName, CubeSegment.class);
                    checkAction.invoke(this, jobSegment);
                    jobCheckActionMap.remove(jobId);
                    jobSegmentMap.remove(jobId);
                }
                jobIds.remove(i);
            }
        }
        try {
            Thread.sleep(5000);
        } catch (InterruptedException e) {
            e.printStackTrace();
        }
    }
}

Example 6

Source File: BuildCubeWithEngine.java From kylin with Apache License 2.0

5 votes

private boolean checkJobState() throws Exception {
    List<String> jobIds = jobService.getAllJobIdsInCache();
    while (true) {
        if (jobIds.size() == 0) {
            return true;
        }
        for (int i = jobIds.size() - 1; i >= 0; i--) {
            String jobId = jobIds.get(i);
            Output job = jobService.getOutputDigest(jobId);
            if (job.getState() == ExecutableState.ERROR) {
                return false;
            } else if (job.getState() == ExecutableState.SUCCEED) {
                String checkActionName = jobCheckActionMap.get(jobId);
                CubeSegment jobSegment = jobSegmentMap.get(jobId);
                if (checkActionName != null) {
                    Method checkAction = this.getClass().getDeclaredMethod(checkActionName, CubeSegment.class);
                    checkAction.invoke(this, jobSegment);
                    jobCheckActionMap.remove(jobId);
                    jobSegmentMap.remove(jobId);
                }
                jobIds.remove(i);
            }
        }
        try {
            Thread.sleep(5000);
        } catch (InterruptedException e) {
            e.printStackTrace();
        }
    }
}

Example 7

Source File: CubingJob.java From Kylin with Apache License 2.0

5 votes

@Override
protected Pair<String, String> formatNotifications(ExecutableState state) {
    final Output output = jobService.getOutput(getId());
    String logMsg;
    switch (output.getState()) {
        case ERROR:
            logMsg = output.getVerboseMsg();
            break;
        case DISCARDED:
            logMsg = "";
            break;
        case SUCCEED:
            logMsg = "";
            break;
        default:
            return null;
    }
    String content = ExecutableConstants.NOTIFY_EMAIL_TEMPLATE;
    content = content.replaceAll("\\$\\{job_name\\}", getName());
    content = content.replaceAll("\\$\\{result\\}", state.toString());
    content = content.replaceAll("\\$\\{cube_name\\}", getCubeName());
    content = content.replaceAll("\\$\\{start_time\\}", new Date(getStartTime()).toString());
    content = content.replaceAll("\\$\\{duration\\}", getDuration() / 60000 + "mins");
    content = content.replaceAll("\\$\\{mr_waiting\\}", getMapReduceWaitTime() / 60000 + "mins");
    content = content.replaceAll("\\$\\{last_update_time\\}", new Date(getLastModified()).toString());
    content = content.replaceAll("\\$\\{submitter\\}", getSubmitter());
    content = content.replaceAll("\\$\\{error_log\\}", logMsg);

    try {
        InetAddress inetAddress = InetAddress.getLocalHost();
        content = content.replaceAll("\\$\\{job_engine\\}", inetAddress.getCanonicalHostName());
    } catch (UnknownHostException e) {
        logger.warn(e.getLocalizedMessage(), e);
    }

    String title = "["+ state.toString() + "] - [Kylin Cube Build Job]-" + getCubeName();
    return Pair.of(title, content);
}

Example 8

Source File: DefaultScheduler.java From Kylin with Apache License 2.0

5 votes

@Override
public void run() {
    // logger.debug("Job Fetcher is running...");
    Map<String, Executable> runningJobs = context.getRunningJobs();
    if (runningJobs.size() >= jobEngineConfig.getMaxConcurrentJobLimit()) {
        logger.warn("There are too many jobs running, Job Fetch will wait until next schedule time");
        return;
    }

    int nRunning = 0, nReady = 0, nOthers = 0;
    for (final String id : executableManager.getAllJobIds()) {
        if (runningJobs.containsKey(id)) {
            // logger.debug("Job id:" + id + " is already running");
            nRunning++;
            continue;
        }
        final Output output = executableManager.getOutput(id);
        if ((output.getState() != ExecutableState.READY)) {
            // logger.debug("Job id:" + id + " not runnable");
            nOthers++;
            continue;
        }
        nReady++;
        AbstractExecutable executable = executableManager.getJob(id);
        String jobDesc = executable.toString();
        logger.info(jobDesc + " prepare to schedule");
        try {
            context.addRunningJob(executable);
            jobPool.execute(new JobRunner(executable));
            logger.info(jobDesc + " scheduled");
        } catch (Exception ex) {
            context.removeRunningJob(executable);
            logger.warn(jobDesc + " fail to schedule", ex);
        }
    }
    logger.info("Job Fetcher: " + nRunning + " running, " + runningJobs.size() + " actual running, " + nReady + " ready, " + nOthers + " others");
}

Example 9

Source File: CubingJob.java From kylin-on-parquet-v2 with Apache License 2.0

4 votes

@Override
protected Pair<String, String> formatNotifications(ExecutableContext context, ExecutableState state) {
    CubeInstance cubeInstance = CubeManager.getInstance(context.getConfig())
            .getCube(CubingExecutableUtil.getCubeName(this.getParams()));
    final Output output = getManager().getOutput(getId());
    if (state != ExecutableState.ERROR
            && !cubeInstance.getDescriptor().getStatusNeedNotify().contains(state.toString())) {
        logger.info("state:" + state + " no need to notify users");
        return null;
    }

    if (!MailNotificationUtil.hasMailNotification(state)) {
        logger.info("Cannot find email template for job state: " + state);
        return null;
    }

    Map<String, Object> dataMap = Maps.newHashMap();
    dataMap.put("job_name", getName());
    dataMap.put("env_name", getDeployEnvName());
    dataMap.put("submitter", StringUtil.noBlank(getSubmitter(), "missing submitter"));
    dataMap.put("job_engine", MailNotificationUtil.getLocalHostName());
    dataMap.put("project_name", getProjectName());
    dataMap.put("cube_name", cubeInstance.getName());
    dataMap.put("source_records_count", String.valueOf(findSourceRecordCount()));
    dataMap.put("start_time", new Date(getStartTime()).toString());
    dataMap.put("duration", getDuration() / 60000 + "mins");
    dataMap.put("mr_waiting", getMapReduceWaitTime() / 60000 + "mins");
    dataMap.put("last_update_time", new Date(getLastModified()).toString());

    if (state == ExecutableState.ERROR) {
        AbstractExecutable errorTask = null;
        Output errorOutput = null;
        for (AbstractExecutable task : getTasks()) {
            errorOutput = getManager().getOutput(task.getId());
            if (errorOutput.getState() == ExecutableState.ERROR) {
                errorTask = task;
                break;
            }
        }
        Preconditions.checkNotNull(errorTask,
                "None of the sub tasks of cubing job " + getId() + " is error and this job should become success.");

        dataMap.put("error_step", errorTask.getName());
        if (errorTask instanceof MapReduceExecutable) {
            final String mrJobId = errorOutput.getExtra().get(ExecutableConstants.MR_JOB_ID);
            dataMap.put("mr_job_id", StringUtil.noBlank(mrJobId, "Not initialized"));
        } else {
            dataMap.put("mr_job_id", MailNotificationUtil.NA);
        }
        dataMap.put("error_log",
                Matcher.quoteReplacement(StringUtil.noBlank(output.getVerboseMsg(), "no error message")));
    }

    String content = MailNotificationUtil.getMailContent(state, dataMap);
    String title = MailNotificationUtil.getMailTitle("JOB", state.toString(), getDeployEnvName(), getProjectName(),
            cubeInstance.getName());
    return Pair.newPair(title, content);
}

Example 10

Source File: DefaultFetcherRunner.java From kylin-on-parquet-v2 with Apache License 2.0

4 votes

@Override
synchronized public void run() {
    try (SetThreadName ignored = new SetThreadName(//
            "FetcherRunner %s", System.identityHashCode(this))) {//
        // logger.debug("Job Fetcher is running...");
        Map<String, Executable> runningJobs = context.getRunningJobs();
        if (isJobPoolFull()) {
            return;
        }

        nRunning = 0;
        nReady = 0;
        nStopped = 0;
        nOthers = 0;
        nError = 0;
        nDiscarded = 0;
        nSUCCEED = 0;
        for (final String id : getExecutableManager().getAllJobIdsInCache()) {
            if (isJobPoolFull()) {
                return;
            }
            if (runningJobs.containsKey(id)) {
                // logger.debug("Job id:" + id + " is already running");
                nRunning++;
                continue;
            }
            if (succeedJobs.contains(id)) {
                nSUCCEED++;
                continue;
            }

            final Output outputDigest;
            try {
                outputDigest = getExecutableManager().getOutputDigest(id);
            } catch (IllegalArgumentException e) {
                logger.warn("job " + id + " output digest is null, skip.", e);
                nOthers++;
                continue;
            }
            if ((outputDigest.getState() != ExecutableState.READY)) {
                // logger.debug("Job id:" + id + " not runnable");
                jobStateCount(id);
                continue;
            }

            final AbstractExecutable executable = getExecutableManager().getJob(id);
            if (executable == null) {
                logger.info("job " + id + " get job is null, skip.");
                nOthers++;
                continue;
            }
            if (!executable.isReady()) {
                nOthers++;
                continue;
            }

            KylinConfig config = jobEngineConfig.getConfig();
            if (config.isSchedulerSafeMode()) {
                String cubeName = executable.getCubeName();
                String projectName = CubeManager.getInstance(config).getCube(cubeName).getProject();
                if (!config.getSafeModeRunnableProjects().contains(projectName) &&
                        executable.getStartTime() == 0) {
                    logger.info("New job is pending for scheduler in safe mode. Project: {}, job: {}",
                            projectName, executable.getName());
                    continue;
                }
            }

            nReady++;
            addToJobPool(executable, executable.getDefaultPriority());
        }

        fetchFailed = false;
        logger.info("Job Fetcher: " + nRunning + " should running, " + runningJobs.size() + " actual running, "
                + nStopped + " stopped, " + nReady + " ready, " + nSUCCEED + " already succeed, " + nError
                + " error, " + nDiscarded + " discarded, " + nOthers + " others");
    } catch (Throwable th) {
        fetchFailed = true; // this could happen when resource store is unavailable
        logger.warn("Job Fetcher caught a exception ", th);
    }
}

Example 11

Source File: CubingJob.java From kylin with Apache License 2.0

4 votes

@Override
protected Pair<String, String> formatNotifications(ExecutableContext context, ExecutableState state) {
    CubeInstance cubeInstance = CubeManager.getInstance(context.getConfig())
            .getCube(CubingExecutableUtil.getCubeName(this.getParams()));
    final Output output = getManager().getOutput(getId());
    if (state != ExecutableState.ERROR
            && !cubeInstance.getDescriptor().getStatusNeedNotify().contains(state.toString())) {
        logger.info("state:" + state + " no need to notify users");
        return null;
    }

    if (!MailNotificationUtil.hasMailNotification(state)) {
        logger.info("Cannot find email template for job state: " + state);
        return null;
    }

    Map<String, Object> dataMap = Maps.newHashMap();
    dataMap.put("job_name", getName());
    dataMap.put("env_name", getDeployEnvName());
    dataMap.put("submitter", StringUtil.noBlank(getSubmitter(), "missing submitter"));
    dataMap.put("job_engine", MailNotificationUtil.getLocalHostName());
    dataMap.put("project_name", getProjectName());
    dataMap.put("cube_name", cubeInstance.getName());
    dataMap.put("source_records_count", String.valueOf(findSourceRecordCount()));
    dataMap.put("start_time", new Date(getStartTime()).toString());
    dataMap.put("duration", getDuration() / 60000 + "mins");
    dataMap.put("mr_waiting", getMapReduceWaitTime() / 60000 + "mins");
    dataMap.put("last_update_time", new Date(getLastModified()).toString());

    if (state == ExecutableState.ERROR) {
        AbstractExecutable errorTask = null;
        Output errorOutput = null;
        for (AbstractExecutable task : getTasks()) {
            errorOutput = getManager().getOutput(task.getId());
            if (errorOutput.getState() == ExecutableState.ERROR) {
                errorTask = task;
                break;
            }
        }
        Preconditions.checkNotNull(errorTask,
                "None of the sub tasks of cubing job " + getId() + " is error and this job should become success.");

        dataMap.put("error_step", errorTask.getName());
        if (errorTask instanceof MapReduceExecutable) {
            final String mrJobId = errorOutput.getExtra().get(ExecutableConstants.MR_JOB_ID);
            dataMap.put("mr_job_id", StringUtil.noBlank(mrJobId, "Not initialized"));
        } else {
            dataMap.put("mr_job_id", MailNotificationUtil.NA);
        }
        dataMap.put("error_log",
                Matcher.quoteReplacement(StringUtil.noBlank(output.getVerboseMsg(), "no error message")));
    }

    String content = MailNotificationUtil.getMailContent(state, dataMap);
    String title = MailNotificationUtil.getMailTitle("JOB",
            state.toString(),
            context.getConfig().getClusterName(),
            getDeployEnvName(),
            getProjectName(),
            cubeInstance.getName());
    return Pair.newPair(title, content);
}

Example 12

Source File: DefaultFetcherRunner.java From kylin with Apache License 2.0

4 votes

@Override
synchronized public void run() {
    try (SetThreadName ignored = new SetThreadName(//
            "FetcherRunner %s", System.identityHashCode(this))) {//
        // logger.debug("Job Fetcher is running...");
        Map<String, Executable> runningJobs = context.getRunningJobs();
        if (isJobPoolFull()) {
            return;
        }

        nRunning = 0;
        nReady = 0;
        nStopped = 0;
        nOthers = 0;
        nError = 0;
        nDiscarded = 0;
        nSUCCEED = 0;
        for (final String id : getExecutableManager().getAllJobIdsInCache()) {
            if (isJobPoolFull()) {
                return;
            }
            if (runningJobs.containsKey(id)) {
                // logger.debug("Job id:" + id + " is already running");
                nRunning++;
                continue;
            }
            if (succeedJobs.contains(id)) {
                nSUCCEED++;
                continue;
            }

            final Output outputDigest;
            try {
                outputDigest = getExecutableManager().getOutputDigest(id);
            } catch (IllegalArgumentException e) {
                logger.warn("job " + id + " output digest is null, skip.", e);
                nOthers++;
                continue;
            }
            if ((outputDigest.getState() != ExecutableState.READY)) {
                // logger.debug("Job id:" + id + " not runnable");
                jobStateCount(id);
                continue;
            }

            final AbstractExecutable executable = getExecutableManager().getJob(id);
            if (executable == null) {
                logger.info("job " + id + " get job is null, skip.");
                nOthers++;
                continue;
            }
            if (!executable.isReady()) {
                nOthers++;
                continue;
            }

            KylinConfig config = jobEngineConfig.getConfig();
            if (config.isSchedulerSafeMode()) {
                String cubeName = executable.getCubeName();
                String projectName = CubeManager.getInstance(config).getCube(cubeName).getProject();
                if (!config.getSafeModeRunnableProjects().contains(projectName) &&
                        executable.getStartTime() == 0) {
                    logger.info("New job is pending for scheduler in safe mode. Project: {}, job: {}",
                            projectName, executable.getName());
                    continue;
                }
            }

            nReady++;
            addToJobPool(executable, executable.getDefaultPriority());
        }

        fetchFailed = false;
        logger.info("Job Fetcher: " + nRunning + " should running, " + runningJobs.size() + " actual running, "
                + nStopped + " stopped, " + nReady + " ready, " + nSUCCEED + " already succeed, " + nError
                + " error, " + nDiscarded + " discarded, " + nOthers + " others");
    } catch (Throwable th) {
        fetchFailed = true; // this could happen when resource store is unavailable
        logger.warn("Job Fetcher caught a exception ", th);
    }
}