Java Code Examples for org.apache.kylin.job.execution.ExecutableState#DISCARDED

The following examples show how to use org.apache.kylin.job.execution.ExecutableState#DISCARDED . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: CubeMetadataUpgrade.java    From Kylin with Apache License 2.0 6 votes vote down vote up 
private ExecutableState parseState(JobStepStatusEnum state) {
    switch (state) {
        case NEW:
        case PENDING:
        case WAITING:
            return ExecutableState.READY;
        case RUNNING:
            return ExecutableState.RUNNING;
        case FINISHED:
            return ExecutableState.SUCCEED;
        case ERROR:
            return ExecutableState.ERROR;
        case DISCARDED:
            return ExecutableState.DISCARDED;
        default:
            return ExecutableState.DISCARDED;
    }

}
Example 2
Source File: FetcherRunner.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up 
protected void jobStateCount(String id) {
    final Output outputDigest = getExecutableManager().getOutputDigest(id);
    // logger.debug("Job id:" + id + " not runnable");
    if (outputDigest.getState() == ExecutableState.SUCCEED) {
        succeedJobs.add(id);
        nSUCCEED++;
    } else if (outputDigest.getState() == ExecutableState.ERROR) {
        nError++;
    } else if (outputDigest.getState() == ExecutableState.DISCARDED) {
        nDiscarded++;
    } else if (outputDigest.getState() == ExecutableState.STOPPED) {
        nStopped++;
    } else {
        if (fetchFailed) {
            getExecutableManager().forceKillJob(id);
            nError++;
        } else {
            nOthers++;
        }
    }
}
Example 3
Source File: CubeMetadataUpgrade.java    From Kylin with Apache License 2.0 6 votes vote down vote up 
private ExecutableState parseState(JobStatusEnum state) {
    switch (state) {
        case NEW:
        case PENDING:
            return ExecutableState.READY;
        case RUNNING:
            return ExecutableState.RUNNING;
        case FINISHED:
            return ExecutableState.SUCCEED;
        case ERROR:
            return ExecutableState.ERROR;
        case DISCARDED:
            return ExecutableState.DISCARDED;
        default:
            return ExecutableState.DISCARDED;
    }
}
Example 4
Source File: CubeService.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up 
protected void releaseAllJobs(CubeInstance cube) {
    final List<CubingJob> cubingJobs = jobService.listJobsByRealizationName(cube.getName(), null);
    for (CubingJob cubingJob : cubingJobs) {
        final ExecutableState status = cubingJob.getStatus();
        if (status != ExecutableState.SUCCEED && status != ExecutableState.DISCARDED) {
            getExecutableManager().discardJob(cubingJob.getId());

            //release global dict lock if exists
            DistributedLock lock = KylinConfig.getInstanceFromEnv().getDistributedLockFactory()
                    .lockForCurrentThread();
            if (lock.isLocked(CubeJobLockUtil.getLockPath(cube.getName(), cubingJob.getId()))) {//release cube job dict lock if exists
                lock.purgeLocks(CubeJobLockUtil.getLockPath(cube.getName(), null));
                logger.info("{} unlock cube job global lock path({}) success", cubingJob.getId(),
                        CubeJobLockUtil.getLockPath(cube.getName(), null));

                if (lock.isLocked(CubeJobLockUtil.getEphemeralLockPath(cube.getName()))) {//release cube job Ephemeral lock if exists
                    lock.purgeLocks(CubeJobLockUtil.getEphemeralLockPath(cube.getName()));
                    logger.info("{} unlock cube job ephemeral lock path({}) success", cubingJob.getId(),
                            CubeJobLockUtil.getEphemeralLockPath(cube.getName()));
                }
            }

        }
    }
}
Example 5
Source File: JobService.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up 
private ExecutableState parseToExecutableState(JobStatusEnum status) {
    Message msg = MsgPicker.getMsg();

    switch (status) {
    case DISCARDED:
        return ExecutableState.DISCARDED;
    case ERROR:
        return ExecutableState.ERROR;
    case FINISHED:
        return ExecutableState.SUCCEED;
    case NEW:
        return ExecutableState.READY;
    case PENDING:
        return ExecutableState.READY;
    case RUNNING:
        return ExecutableState.RUNNING;
    case STOPPED:
        return ExecutableState.STOPPED;
    default:
        throw new BadRequestException(String.format(Locale.ROOT, msg.getILLEGAL_EXECUTABLE_STATE(), status));
    }
}
Example 6
Source File: JobService.java    From kylin with Apache License 2.0 6 votes vote down vote up 
private ExecutableState parseToExecutableState(JobStatusEnum status) {
    Message msg = MsgPicker.getMsg();

    switch (status) {
    case DISCARDED:
        return ExecutableState.DISCARDED;
    case ERROR:
        return ExecutableState.ERROR;
    case FINISHED:
        return ExecutableState.SUCCEED;
    case NEW:
        return ExecutableState.READY;
    case PENDING:
        return ExecutableState.READY;
    case RUNNING:
        return ExecutableState.RUNNING;
    case STOPPED:
        return ExecutableState.STOPPED;
    default:
        throw new BadRequestException(String.format(Locale.ROOT, msg.getILLEGAL_EXECUTABLE_STATE(), status));
    }
}
Example 7
Source File: CubeService.java    From kylin with Apache License 2.0 6 votes vote down vote up 
protected void releaseAllJobs(CubeInstance cube) {
    final List<CubingJob> cubingJobs = jobService.listJobsByRealizationName(cube.getName(), null);
    for (CubingJob cubingJob : cubingJobs) {
        final ExecutableState status = cubingJob.getStatus();
        if (status != ExecutableState.SUCCEED && status != ExecutableState.DISCARDED) {
            getExecutableManager().discardJob(cubingJob.getId());

            //release global dict lock if exists
            DistributedLock lock = KylinConfig.getInstanceFromEnv().getDistributedLockFactory()
                    .lockForCurrentThread();
            if (lock.isLocked(CubeJobLockUtil.getLockPath(cube.getName(), cubingJob.getId()))) {//release cube job dict lock if exists
                lock.purgeLocks(CubeJobLockUtil.getLockPath(cube.getName(), null));
                logger.info("{} unlock cube job global lock path({}) success", cubingJob.getId(),
                        CubeJobLockUtil.getLockPath(cube.getName(), null));

                if (lock.isLocked(CubeJobLockUtil.getEphemeralLockPath(cube.getName()))) {//release cube job Ephemeral lock if exists
                    lock.purgeLocks(CubeJobLockUtil.getEphemeralLockPath(cube.getName()));
                    logger.info("{} unlock cube job ephemeral lock path({}) success", cubingJob.getId(),
                            CubeJobLockUtil.getEphemeralLockPath(cube.getName()));
                }
            }

        }
    }
}
Example 8
Source File: JobService.java    From Kylin with Apache License 2.0 6 votes vote down vote up 
private ExecutableState parseToExecutableState(JobStatusEnum status) {
    switch (status) {
        case DISCARDED:
            return ExecutableState.DISCARDED;
        case ERROR:
            return ExecutableState.ERROR;
        case FINISHED:
            return ExecutableState.SUCCEED;
        case NEW:
            return ExecutableState.READY;
        case PENDING:
            return ExecutableState.READY;
        case RUNNING:
            return ExecutableState.RUNNING;
        default:
            throw new RuntimeException("illegal status:" + status);
    }
}
Example 9
Source File: FetcherRunner.java    From kylin with Apache License 2.0 6 votes vote down vote up 
protected void jobStateCount(String id) {
    final Output outputDigest = getExecutableManager().getOutputDigest(id);
    // logger.debug("Job id:" + id + " not runnable");
    if (outputDigest.getState() == ExecutableState.SUCCEED) {
        succeedJobs.add(id);
        nSUCCEED++;
    } else if (outputDigest.getState() == ExecutableState.ERROR) {
        nError++;
    } else if (outputDigest.getState() == ExecutableState.DISCARDED) {
        nDiscarded++;
    } else if (outputDigest.getState() == ExecutableState.STOPPED) {
        nStopped++;
    } else {
        if (fetchFailed) {
            getExecutableManager().forceKillJob(id);
            nError++;
        } else {
            nOthers++;
        }
    }
}
Example 10
Source File: JobInfoConverterTest.java    From kylin with Apache License 2.0 5 votes vote down vote up 
@Test
public void testParseToJobStepStatusReturnsJobStepStatusDiscarded() {
    ExecutableState executableState = ExecutableState.DISCARDED;
    JobStepStatusEnum jobStepStatusEnum = JobInfoConverter.parseToJobStepStatus(executableState);

    assertTrue(jobStepStatusEnum.isComplete());
    assertFalse(jobStepStatusEnum.isRunable());
    assertEquals(16, jobStepStatusEnum.getCode());
    assertEquals(JobStepStatusEnum.DISCARDED, jobStepStatusEnum);
}
Example 11
Source File: CubeService.java    From Kylin with Apache License 2.0 5 votes vote down vote up 
/**
 * purge the cube
 *
 * @throws IOException
 * @throws JobException
 * @throws CubeIntegrityException
 */
private void releaseAllSegments(CubeInstance cube) throws IOException, JobException {
    final List<CubingJob> cubingJobs = listAllCubingJobs(cube.getName(), null);
    for (CubingJob cubingJob : cubingJobs) {
        final ExecutableState status = cubingJob.getStatus();
        if (status != ExecutableState.SUCCEED && status != ExecutableState.STOPPED && status != ExecutableState.DISCARDED) {
            getExecutableManager().discardJob(cubingJob.getId());
        }
    }
    cube.getSegments().clear();
    CubeManager.getInstance(getConfig()).updateCube(cube);
}
Example 12
Source File: BaseSchedulerTest.java    From Kylin with Apache License 2.0 5 votes vote down vote up 
protected void waitForJobFinish(String jobId) {
    while (true) {
        AbstractExecutable job = jobService.getJob(jobId);
        final ExecutableState status = job.getStatus();
        if (status == ExecutableState.SUCCEED || status == ExecutableState.ERROR || status == ExecutableState.STOPPED || status == ExecutableState.DISCARDED) {
            break;
        } else {
            try {
                Thread.sleep(5000);
            } catch (InterruptedException e) {
                e.printStackTrace();
            }
        }
    }
}
Example 13
Source File: BaseSchedulerTest.java    From kylin with Apache License 2.0 5 votes vote down vote up 
protected void waitForJobFinish(String jobId, int maxWaitTime) {
    int error = 0;
    long start = System.currentTimeMillis();
    final int errorLimit = 3;
    while (error < errorLimit && (System.currentTimeMillis() - start < maxWaitTime)) {
        try {
            Thread.sleep(1000);
        } catch (InterruptedException e) {
            e.printStackTrace();
        }

        try {
            AbstractExecutable job = execMgr.getJob(jobId);
            ExecutableState status = job.getStatus();
            if (status == ExecutableState.SUCCEED || status == ExecutableState.ERROR
                    || status == ExecutableState.STOPPED || status == ExecutableState.DISCARDED) {
                break;
            }
        } catch (Exception ex) {
            logger.error("", ex);
            error++;
        }
    }

    if (error >= errorLimit) {
        throw new RuntimeException("too many exceptions");
    }

    if (System.currentTimeMillis() - start >= maxWaitTime) {
        throw new RuntimeException("too long wait time");
    }
}
Example 14
Source File: BuildCubeWithStream.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
protected void waitForJob(String jobId) {
    while (true) {
        AbstractExecutable job = jobService.getJob(jobId);
        if (job.getStatus() == ExecutableState.SUCCEED || job.getStatus() == ExecutableState.ERROR
                || job.getStatus() == ExecutableState.DISCARDED) {
            break;
        } else {
            try {
                Thread.sleep(5000);
            } catch (InterruptedException e) {
                e.printStackTrace();
            }
        }
    }
}
Example 15
Source File: BaseTestDistributedScheduler.java    From kylin with Apache License 2.0 5 votes vote down vote up 
void waitForJobFinish(String jobId) {
    while (true) {
        AbstractExecutable job = execMgr.getJob(jobId);
        final ExecutableState status = job.getStatus();
        if (status == ExecutableState.SUCCEED || status == ExecutableState.ERROR || status == ExecutableState.STOPPED || status == ExecutableState.DISCARDED) {
            break;
        } else {
            try {
                Thread.sleep(1000);
            } catch (InterruptedException e) {
                e.printStackTrace();
            }
        }
    }
}
Example 16
Source File: BuildCubeWithStream.java    From kylin with Apache License 2.0 5 votes vote down vote up 
protected void waitForJob(String jobId) {
    while (true) {
        AbstractExecutable job = jobService.getJob(jobId);
        if (job.getStatus() == ExecutableState.SUCCEED || job.getStatus() == ExecutableState.ERROR
                || job.getStatus() == ExecutableState.DISCARDED) {
            break;
        } else {
            try {
                Thread.sleep(5000);
            } catch (InterruptedException e) {
                e.printStackTrace();
            }
        }
    }
}
Example 17
Source File: BaseSchedulerTest.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
protected void waitForJobFinish(String jobId, int maxWaitTime) {
    int error = 0;
    long start = System.currentTimeMillis();
    final int errorLimit = 3;
    while (error < errorLimit && (System.currentTimeMillis() - start < maxWaitTime)) {
        try {
            Thread.sleep(1000);
        } catch (InterruptedException e) {
            e.printStackTrace();
        }

        try {
            AbstractExecutable job = execMgr.getJob(jobId);
            ExecutableState status = job.getStatus();
            if (status == ExecutableState.SUCCEED || status == ExecutableState.ERROR
                    || status == ExecutableState.STOPPED || status == ExecutableState.DISCARDED) {
                break;
            }
        } catch (Exception ex) {
            logger.error("", ex);
            error++;
        }
    }

    if (error >= errorLimit) {
        throw new RuntimeException("too many exceptions");
    }

    if (System.currentTimeMillis() - start >= maxWaitTime) {
        throw new RuntimeException("too long wait time");
    }
}
Example 18
Source File: JobInfoConverterTest.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
@Test
public void testParseToJobStatusReturnsJobStatusDiscarded() {
    ExecutableState executableState = ExecutableState.DISCARDED;
    JobStatusEnum jobStatusEnum = JobInfoConverter.parseToJobStatus(executableState);

    assertEquals(16, jobStatusEnum.getCode());
    assertEquals(JobStatusEnum.DISCARDED, jobStatusEnum);
}
Example 19
Source File: JobInfoConverterTest.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
@Test
public void testParseToJobStepStatusReturnsJobStepStatusDiscarded() {
    ExecutableState executableState = ExecutableState.DISCARDED;
    JobStepStatusEnum jobStepStatusEnum = JobInfoConverter.parseToJobStepStatus(executableState);

    assertTrue(jobStepStatusEnum.isComplete());
    assertFalse(jobStepStatusEnum.isRunable());
    assertEquals(16, jobStepStatusEnum.getCode());
    assertEquals(JobStepStatusEnum.DISCARDED, jobStepStatusEnum);
}
Example 20
Source File: BaseTestDistributedScheduler.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
void waitForJobFinish(String jobId) {
    while (true) {
        AbstractExecutable job = execMgr.getJob(jobId);
        final ExecutableState status = job.getStatus();
        if (status == ExecutableState.SUCCEED || status == ExecutableState.ERROR || status == ExecutableState.STOPPED || status == ExecutableState.DISCARDED) {
            break;
        } else {
            try {
                Thread.sleep(1000);
            } catch (InterruptedException e) {
                e.printStackTrace();
            }
        }
    }
}