org.apache.kylin.cube.model.CubeBuildTypeEnum Java Examples
The following examples show how to use
org.apache.kylin.cube.model.CubeBuildTypeEnum.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
| Source File: JobService.java From Kylin with Apache License 2.0 | 6 votes |
|
private JobInstance parseToJobInstance(AbstractExecutable job) {
if (job == null) {
return null;
}
Preconditions.checkState(job instanceof CubingJob, "illegal job type, id:" + job.getId());
CubingJob cubeJob = (CubingJob) job;
final JobInstance result = new JobInstance();
result.setName(job.getName());
result.setRelatedCube(cubeJob.getCubeName());
result.setRelatedSegment(cubeJob.getSegmentId());
result.setLastModified(cubeJob.getLastModified());
result.setSubmitter(cubeJob.getSubmitter());
result.setUuid(cubeJob.getId());
result.setType(CubeBuildTypeEnum.BUILD);
result.setStatus(parseToJobStatus(job.getStatus()));
result.setMrWaiting(cubeJob.getMapReduceWaitTime() / 1000);
result.setDuration(cubeJob.getDuration() / 1000);
for (int i = 0; i < cubeJob.getTasks().size(); ++i) {
AbstractExecutable task = cubeJob.getTasks().get(i);
result.addStep(parseToJobStep(task, i));
}
return result;
}
Example #2
| Source File: JobInfoConverterTest.java From kylin with Apache License 2.0 | 6 votes |
|
@Test
public void testParseToJobInstance4CheckpointJob() {
Test2Job task = new Test2Job();
String jobId = UUID.randomUUID().toString();
String cubeName = "cube1";
task.setId(jobId);
task.setParam(CubingExecutableUtil.CUBE_NAME, cubeName);
Map<String, Output> outPutMap = Maps.newHashMap();
DefaultOutput executeOutput = new DefaultOutput();
executeOutput.setState(ExecutableState.READY);
Map<String, String> extraMap = Maps.newHashMap();
executeOutput.setExtra(extraMap);
outPutMap.put(jobId, executeOutput);
JobInstance instance3 = JobInfoConverter.parseToJobInstanceQuietly(task, outPutMap);
// no exception thrown is expected
assertEquals(jobId, instance3.getId());
assertEquals(CubeBuildTypeEnum.CHECKPOINT, instance3.getType());
assertEquals(cubeName, instance3.getRelatedCube());
assertEquals(JobStatusEnum.PENDING, instance3.getStatus());
}
Example #3
| Source File: JobInfoConverterTest.java From kylin with Apache License 2.0 | 6 votes |
|
@Test
public void testParseToJobInstance4CuboidJob() {
TestJob task = new TestJob();
String jobId = UUID.randomUUID().toString();
String cubeName = "cube1";
task.setId(jobId);
task.setParam(CubingExecutableUtil.CUBE_NAME, cubeName);
Map<String, Output> outPutMap = Maps.newHashMap();
DefaultOutput executeOutput = new DefaultOutput();
executeOutput.setState(ExecutableState.READY);
Map<String, String> extraMap = Maps.newHashMap();
executeOutput.setExtra(extraMap);
outPutMap.put(jobId, executeOutput);
JobInstance instance3 = JobInfoConverter.parseToJobInstanceQuietly(task, outPutMap);
// no exception thrown is expected
assertEquals(jobId, instance3.getId());
assertEquals(CubeBuildTypeEnum.BUILD, instance3.getType());
assertEquals(cubeName, instance3.getRelatedCube());
assertEquals(JobStatusEnum.PENDING, instance3.getStatus());
}
Example #4
| Source File: JobInfoConverter.java From kylin with Apache License 2.0 | 5 votes |
|
public static JobInstance parseToJobInstance(CheckpointExecutable job, Map<String, Output> outputs) {
if (job == null) {
logger.warn("job is null.");
return null;
}
Output output = outputs.get(job.getId());
if (output == null) {
logger.warn("job output is null.");
return null;
}
final JobInstance result = new JobInstance();
result.setName(job.getName());
result.setProjectName(job.getProjectName());
result.setRelatedCube(CubingExecutableUtil.getCubeName(job.getParams()));
result.setDisplayCubeName(CubingExecutableUtil.getCubeName(job.getParams()));
result.setLastModified(output.getLastModified());
result.setSubmitter(job.getSubmitter());
result.setUuid(job.getId());
result.setType(CubeBuildTypeEnum.CHECKPOINT);
result.setStatus(parseToJobStatus(output.getState()));
result.setBuildInstance(AbstractExecutable.getBuildInstance(output));
result.setExecStartTime(AbstractExecutable.getStartTime(output));
result.setExecEndTime(AbstractExecutable.getEndTime(output));
result.setExecInterruptTime(AbstractExecutable.getInterruptTime(output));
result.setDuration(AbstractExecutable.getDuration(result.getExecStartTime(), result.getExecEndTime(),
result.getExecInterruptTime()) / 1000);
for (int i = 0; i < job.getTasks().size(); ++i) {
AbstractExecutable task = job.getTasks().get(i);
result.addStep(parseToJobStep(task, i, outputs.get(task.getId())));
}
return result;
}
Example #5
| Source File: JobService.java From kylin with Apache License 2.0 | 5 votes |
|
protected JobInstance getCheckpointJobInstance(AbstractExecutable job) {
Message msg = MsgPicker.getMsg();
if (job == null) {
return null;
}
if (!(job instanceof CheckpointExecutable)) {
throw new BadRequestException(String.format(Locale.ROOT, msg.getILLEGAL_JOB_TYPE(), job.getId()));
}
CheckpointExecutable checkpointExecutable = (CheckpointExecutable) job;
Output output = checkpointExecutable.getOutput();
final JobInstance result = new JobInstance();
result.setName(job.getName());
result.setProjectName(checkpointExecutable.getProjectName());
result.setRelatedCube(CubingExecutableUtil.getCubeName(job.getParams()));
result.setDisplayCubeName(CubingExecutableUtil.getCubeName(job.getParams()));
result.setLastModified(job.getLastModified());
result.setSubmitter(job.getSubmitter());
result.setUuid(job.getId());
result.setExecStartTime(job.getStartTime());
result.setExecEndTime(job.getEndTime());
result.setExecInterruptTime(job.getInterruptTime());
result.setType(CubeBuildTypeEnum.CHECKPOINT);
result.setStatus(JobInfoConverter.parseToJobStatus(job.getStatus()));
result.setBuildInstance(AbstractExecutable.getBuildInstance(output));
result.setDuration(job.getDuration() / 1000);
for (int i = 0; i < checkpointExecutable.getTasks().size(); ++i) {
AbstractExecutable task = checkpointExecutable.getTasks().get(i);
result.addStep(JobInfoConverter.parseToJobStep(task, i, getExecutableManager().getOutput(task.getId())));
}
return result;
}
Example #6
| Source File: JobService.java From kylin with Apache License 2.0 | 5 votes |
|
protected JobInstance getLookupSnapshotBuildJobInstance(LookupSnapshotBuildJob job) {
if (job == null) {
return null;
}
Output output = job.getOutput();
final JobInstance result = new JobInstance();
result.setName(job.getName());
result.setProjectName(job.getProjectName());
result.setRelatedCube(CubingExecutableUtil.getCubeName(job.getParams()));
result.setRelatedSegment(CubingExecutableUtil.getSegmentId(job.getParams()));
result.setRelatedSegmentName(CubingExecutableUtil.getSegmentName(job.getParams()));
result.setLastModified(job.getLastModified());
result.setSubmitter(job.getSubmitter());
result.setUuid(job.getId());
result.setExecStartTime(job.getStartTime());
result.setExecEndTime(job.getEndTime());
result.setExecInterruptTime(job.getInterruptTime());
result.setType(CubeBuildTypeEnum.BUILD);
result.setStatus(JobInfoConverter.parseToJobStatus(job.getStatus()));
result.setBuildInstance(AbstractExecutable.getBuildInstance(output));
result.setDuration(job.getDuration() / 1000);
for (int i = 0; i < job.getTasks().size(); ++i) {
AbstractExecutable task = job.getTasks().get(i);
result.addStep(JobInfoConverter.parseToJobStep(task, i, getExecutableManager().getOutput(task.getId())));
}
return result;
}
Example #7
| Source File: JobService.java From kylin with Apache License 2.0 | 5 votes |
|
public JobInstance submitJob(CubeInstance cube, TSRange tsRange, SegmentRange segRange, //
Map<Integer, Long> sourcePartitionOffsetStart, Map<Integer, Long> sourcePartitionOffsetEnd,
CubeBuildTypeEnum buildType, boolean force, String submitter, Integer priorityOffset) throws IOException {
aclEvaluate.checkProjectOperationPermission(cube);
JobInstance jobInstance = submitJobInternal(cube, tsRange, segRange, sourcePartitionOffsetStart,
sourcePartitionOffsetEnd, buildType, force, submitter, priorityOffset);
return jobInstance;
}
Example #8
| Source File: CubeController.java From kylin with Apache License 2.0 | 5 votes |
|
private JobInstance buildInternal(String cubeName, TSRange tsRange, SegmentRange segRange, //
Map<Integer, Long> sourcePartitionOffsetStart, Map<Integer, Long> sourcePartitionOffsetEnd,
String buildType, boolean force, Integer priorityOffset) {
try {
String submitter = SecurityContextHolder.getContext().getAuthentication().getName();
CubeInstance cube = jobService.getCubeManager().getCube(cubeName);
checkBuildingSegment(cube);
return jobService.submitJob(cube, tsRange, segRange, sourcePartitionOffsetStart, sourcePartitionOffsetEnd,
CubeBuildTypeEnum.valueOf(buildType), force, submitter, priorityOffset);
} catch (Throwable e) {
logger.error(e.getLocalizedMessage(), e);
throw new InternalErrorException(e.getLocalizedMessage(), e);
}
}
Example #9
| Source File: CubeBuildingCLI.java From kylin-on-parquet-v2 with Apache License 2.0 | 5 votes |
|
private void run(String cubeName, long startDate, long endDate, String buildType) throws IOException, JobException {
CubeInstance cube = cubeManager.getCube(cubeName);
Preconditions.checkArgument(cube != null, "Cube named " + cubeName + " does not exist!!!");
CubeBuildTypeEnum buildTypeEnum = CubeBuildTypeEnum.valueOf(buildType);
Preconditions.checkArgument(buildTypeEnum != null, "Build type named " + buildType + " does not exist!!!");
submitJob(cube, new TSRange(startDate, endDate), buildTypeEnum, false, "SYSTEM");
}
Example #10
| Source File: JobInstanceExtractor.java From kylin with Apache License 2.0 | 5 votes |
|
private JobInstance parseToJobInstance(CubingJob cubeJob, Map<String, Output> outputs) {
CubeInstance cube = CubeManager.getInstance(KylinConfig.getInstanceFromEnv())
.getCube(CubingExecutableUtil.getCubeName(cubeJob.getParams()));
Output output = outputs.get(cubeJob.getId());
final JobInstance result = new JobInstance();
result.setName(cubeJob.getName());
result.setProjectName(cubeJob.getProjectName());
if (cube != null) {
result.setRelatedCube(cube.getName());
result.setDisplayCubeName(cube.getDisplayName());
} else {
String cubeName = CubingExecutableUtil.getCubeName(cubeJob.getParams());
result.setRelatedCube(cubeName);
result.setDisplayCubeName(cubeName);
}
result.setRelatedSegment(CubingExecutableUtil.getSegmentId(cubeJob.getParams()));
result.setRelatedSegmentName(CubingExecutableUtil.getSegmentName(cubeJob.getParams()));
result.setLastModified(output.getLastModified());
result.setSubmitter(cubeJob.getSubmitter());
result.setUuid(cubeJob.getId());
result.setType(CubeBuildTypeEnum.BUILD);
result.setStatus(parseToJobStatus(output.getState()));
result.setMrWaiting(AbstractExecutable.getExtraInfoAsLong(output, CubingJob.MAP_REDUCE_WAIT_TIME, 0L) / 1000);
result.setBuildInstance(AbstractExecutable.getBuildInstance(output));
result.setExecStartTime(AbstractExecutable.getStartTime(output));
result.setExecEndTime(AbstractExecutable.getEndTime(output));
result.setExecInterruptTime(AbstractExecutable.getInterruptTime(output));
result.setDuration(AbstractExecutable.getDuration(AbstractExecutable.getStartTime(output), AbstractExecutable.getEndTime(output), AbstractExecutable.getInterruptTime(output)) / 1000);
for (int i = 0; i < cubeJob.getTasks().size(); ++i) {
AbstractExecutable task = cubeJob.getTasks().get(i);
result.addStep(parseToJobStep(task, i, outputs.get(task.getId())));
}
return result;
}
Example #11
| Source File: CubeBuildingCLI.java From kylin with Apache License 2.0 | 5 votes |
|
private void run(String cubeName, long startDate, long endDate, String buildType) throws IOException, JobException {
CubeInstance cube = cubeManager.getCube(cubeName);
Preconditions.checkArgument(cube != null, "Cube named " + cubeName + " does not exist!!!");
CubeBuildTypeEnum buildTypeEnum = CubeBuildTypeEnum.valueOf(buildType);
Preconditions.checkArgument(buildTypeEnum != null, "Build type named " + buildType + " does not exist!!!");
submitJob(cube, new TSRange(startDate, endDate), buildTypeEnum, false, "SYSTEM");
}
Example #12
| Source File: JobService.java From kylin-on-parquet-v2 with Apache License 2.0 | 5 votes |
|
protected JobInstance getLookupSnapshotBuildJobInstance(LookupSnapshotBuildJob job) {
if (job == null) {
return null;
}
Output output = job.getOutput();
final JobInstance result = new JobInstance();
result.setName(job.getName());
result.setProjectName(job.getProjectName());
result.setRelatedCube(CubingExecutableUtil.getCubeName(job.getParams()));
result.setRelatedSegment(CubingExecutableUtil.getSegmentId(job.getParams()));
result.setRelatedSegmentName(CubingExecutableUtil.getSegmentName(job.getParams()));
result.setLastModified(job.getLastModified());
result.setSubmitter(job.getSubmitter());
result.setUuid(job.getId());
result.setExecStartTime(job.getStartTime());
result.setExecEndTime(job.getEndTime());
result.setExecInterruptTime(job.getInterruptTime());
result.setType(CubeBuildTypeEnum.BUILD);
result.setStatus(JobInfoConverter.parseToJobStatus(job.getStatus()));
result.setBuildInstance(AbstractExecutable.getBuildInstance(output));
result.setDuration(job.getDuration() / 1000);
for (int i = 0; i < job.getTasks().size(); ++i) {
AbstractExecutable task = job.getTasks().get(i);
result.addStep(JobInfoConverter.parseToJobStep(task, i, getExecutableManager().getOutput(task.getId())));
}
return result;
}
Example #13
| Source File: JobInstanceExtractor.java From kylin-on-parquet-v2 with Apache License 2.0 | 5 votes |
|
private JobInstance parseToJobInstance(CubingJob cubeJob, Map<String, Output> outputs) {
CubeInstance cube = CubeManager.getInstance(KylinConfig.getInstanceFromEnv())
.getCube(CubingExecutableUtil.getCubeName(cubeJob.getParams()));
Output output = outputs.get(cubeJob.getId());
final JobInstance result = new JobInstance();
result.setName(cubeJob.getName());
result.setProjectName(cubeJob.getProjectName());
if (cube != null) {
result.setRelatedCube(cube.getName());
result.setDisplayCubeName(cube.getDisplayName());
} else {
String cubeName = CubingExecutableUtil.getCubeName(cubeJob.getParams());
result.setRelatedCube(cubeName);
result.setDisplayCubeName(cubeName);
}
result.setRelatedSegment(CubingExecutableUtil.getSegmentId(cubeJob.getParams()));
result.setRelatedSegmentName(CubingExecutableUtil.getSegmentName(cubeJob.getParams()));
result.setLastModified(output.getLastModified());
result.setSubmitter(cubeJob.getSubmitter());
result.setUuid(cubeJob.getId());
result.setType(CubeBuildTypeEnum.BUILD);
result.setStatus(parseToJobStatus(output.getState()));
result.setMrWaiting(AbstractExecutable.getExtraInfoAsLong(output, CubingJob.MAP_REDUCE_WAIT_TIME, 0L) / 1000);
result.setBuildInstance(AbstractExecutable.getBuildInstance(output));
result.setExecStartTime(AbstractExecutable.getStartTime(output));
result.setExecEndTime(AbstractExecutable.getEndTime(output));
result.setExecInterruptTime(AbstractExecutable.getInterruptTime(output));
result.setDuration(AbstractExecutable.getDuration(AbstractExecutable.getStartTime(output), AbstractExecutable.getEndTime(output), AbstractExecutable.getInterruptTime(output)) / 1000);
for (int i = 0; i < cubeJob.getTasks().size(); ++i) {
AbstractExecutable task = cubeJob.getTasks().get(i);
result.addStep(parseToJobStep(task, i, outputs.get(task.getId())));
}
return result;
}
Example #14
| Source File: JobInfoConverter.java From kylin-on-parquet-v2 with Apache License 2.0 | 5 votes |
|
public static JobInstance parseToJobInstance(CheckpointExecutable job, Map<String, Output> outputs) {
if (job == null) {
logger.warn("job is null.");
return null;
}
Output output = outputs.get(job.getId());
if (output == null) {
logger.warn("job output is null.");
return null;
}
final JobInstance result = new JobInstance();
result.setName(job.getName());
result.setProjectName(job.getProjectName());
result.setRelatedCube(CubingExecutableUtil.getCubeName(job.getParams()));
result.setDisplayCubeName(CubingExecutableUtil.getCubeName(job.getParams()));
result.setLastModified(output.getLastModified());
result.setSubmitter(job.getSubmitter());
result.setUuid(job.getId());
result.setType(CubeBuildTypeEnum.CHECKPOINT);
result.setStatus(parseToJobStatus(output.getState()));
result.setBuildInstance(AbstractExecutable.getBuildInstance(output));
result.setExecStartTime(AbstractExecutable.getStartTime(output));
result.setExecEndTime(AbstractExecutable.getEndTime(output));
result.setExecInterruptTime(AbstractExecutable.getInterruptTime(output));
result.setDuration(AbstractExecutable.getDuration(result.getExecStartTime(), result.getExecEndTime(),
result.getExecInterruptTime()) / 1000);
for (int i = 0; i < job.getTasks().size(); ++i) {
AbstractExecutable task = job.getTasks().get(i);
result.addStep(parseToJobStep(task, i, outputs.get(task.getId())));
}
return result;
}
Example #15
| Source File: CubeController.java From kylin-on-parquet-v2 with Apache License 2.0 | 5 votes |
|
private JobInstance buildInternal(String cubeName, TSRange tsRange, SegmentRange segRange, //
Map<Integer, Long> sourcePartitionOffsetStart, Map<Integer, Long> sourcePartitionOffsetEnd,
String buildType, boolean force, Integer priorityOffset) {
try {
String submitter = SecurityContextHolder.getContext().getAuthentication().getName();
CubeInstance cube = jobService.getCubeManager().getCube(cubeName);
checkBuildingSegment(cube);
return jobService.submitJob(cube, tsRange, segRange, sourcePartitionOffsetStart, sourcePartitionOffsetEnd,
CubeBuildTypeEnum.valueOf(buildType), force, submitter, priorityOffset);
} catch (Throwable e) {
logger.error(e.getLocalizedMessage(), e);
throw new InternalErrorException(e.getLocalizedMessage(), e);
}
}
Example #16
| Source File: JobService.java From kylin-on-parquet-v2 with Apache License 2.0 | 5 votes |
|
public JobInstance submitJob(CubeInstance cube, TSRange tsRange, SegmentRange segRange, //
Map<Integer, Long> sourcePartitionOffsetStart, Map<Integer, Long> sourcePartitionOffsetEnd,
CubeBuildTypeEnum buildType, boolean force, String submitter, Integer priorityOffset) throws IOException {
aclEvaluate.checkProjectOperationPermission(cube);
JobInstance jobInstance = submitJobInternal(cube, tsRange, segRange, sourcePartitionOffsetStart,
sourcePartitionOffsetEnd, buildType, force, submitter, priorityOffset);
return jobInstance;
}
Example #17
| Source File: JobService.java From kylin-on-parquet-v2 with Apache License 2.0 | 5 votes |
|
protected JobInstance getCheckpointJobInstance(AbstractExecutable job) {
Message msg = MsgPicker.getMsg();
if (job == null) {
return null;
}
if (!(job instanceof CheckpointExecutable)) {
throw new BadRequestException(String.format(Locale.ROOT, msg.getILLEGAL_JOB_TYPE(), job.getId()));
}
CheckpointExecutable checkpointExecutable = (CheckpointExecutable) job;
Output output = checkpointExecutable.getOutput();
final JobInstance result = new JobInstance();
result.setName(job.getName());
result.setProjectName(checkpointExecutable.getProjectName());
result.setRelatedCube(CubingExecutableUtil.getCubeName(job.getParams()));
result.setDisplayCubeName(CubingExecutableUtil.getCubeName(job.getParams()));
result.setLastModified(job.getLastModified());
result.setSubmitter(job.getSubmitter());
result.setUuid(job.getId());
result.setExecStartTime(job.getStartTime());
result.setExecEndTime(job.getEndTime());
result.setExecInterruptTime(job.getInterruptTime());
result.setType(CubeBuildTypeEnum.CHECKPOINT);
result.setStatus(JobInfoConverter.parseToJobStatus(job.getStatus()));
result.setBuildInstance(AbstractExecutable.getBuildInstance(output));
result.setDuration(job.getDuration() / 1000);
for (int i = 0; i < checkpointExecutable.getTasks().size(); ++i) {
AbstractExecutable task = checkpointExecutable.getTasks().get(i);
result.addStep(JobInfoConverter.parseToJobStep(task, i, getExecutableManager().getOutput(task.getId())));
}
return result;
}
Example #18
| Source File: JobService.java From kylin-on-parquet-v2 with Apache License 2.0 | 4 votes |
|
protected JobInstance getSingleJobInstance(AbstractExecutable job) {
Message msg = MsgPicker.getMsg();
if (job == null) {
return null;
}
DefaultChainedExecutable cubeJob;
if (job instanceof CubingJob) {
cubeJob = (CubingJob) job;
} else {
throw new BadRequestException(String.format(Locale.ROOT, msg.getILLEGAL_JOB_TYPE(), job.getId()));
}
CubeInstance cube = CubeManager.getInstance(KylinConfig.getInstanceFromEnv())
.getCube(CubingExecutableUtil.getCubeName(cubeJob.getParams()));
Output output = cubeJob.getOutput();
final JobInstance result = new JobInstance();
result.setName(job.getName());
result.setProjectName(cubeJob.getProjectName());
if (cube != null) {
result.setRelatedCube(cube.getName());
result.setDisplayCubeName(cube.getDisplayName());
} else {
String cubeName = CubingExecutableUtil.getCubeName(cubeJob.getParams());
result.setRelatedCube(cubeName);
result.setDisplayCubeName(cubeName);
}
result.setRelatedSegment(CubingExecutableUtil.getSegmentId(cubeJob.getParams()));
result.setRelatedSegmentName(CubingExecutableUtil.getSegmentName(cubeJob.getParams()));
result.setLastModified(cubeJob.getLastModified());
result.setSubmitter(cubeJob.getSubmitter());
result.setUuid(cubeJob.getId());
result.setExecStartTime(cubeJob.getStartTime());
result.setExecEndTime(cubeJob.getEndTime());
result.setExecInterruptTime(cubeJob.getInterruptTime());
result.setType(CubeBuildTypeEnum.BUILD);
result.setStatus(JobInfoConverter.parseToJobStatus(job.getStatus()));
result.setMrWaiting(cubeJob.getMapReduceWaitTime() / 1000);
result.setBuildInstance(AbstractExecutable.getBuildInstance(output));
result.setDuration(cubeJob.getDuration() / 1000);
for (int i = 0; i < cubeJob.getTasks().size(); ++i) {
AbstractExecutable task = cubeJob.getTasks().get(i);
result.addStep(JobInfoConverter.parseToJobStep(task, i, getExecutableManager().getOutput(task.getId())));
}
return result;
}
Example #19
| Source File: JobInstance.java From kylin with Apache License 2.0 | 4 votes |
|
public CubeBuildTypeEnum getType() {
return type;
}
Example #20
| Source File: JobInstance.java From kylin with Apache License 2.0 | 4 votes |
|
public void setType(CubeBuildTypeEnum type) {
this.type = type;
}
Example #21
| Source File: JobInstance.java From kylin-on-parquet-v2 with Apache License 2.0 | 4 votes |
|
public void setType(CubeBuildTypeEnum type) {
this.type = type;
}
Example #22
| Source File: JobInstance.java From kylin-on-parquet-v2 with Apache License 2.0 | 4 votes |
|
public CubeBuildTypeEnum getType() {
return type;
}
Example #23
| Source File: JobService.java From kylin with Apache License 2.0 | 4 votes |
|
public JobInstance submitJobInternal(CubeInstance cube, TSRange tsRange, SegmentRange segRange, //
Map<Integer, Long> sourcePartitionOffsetStart, Map<Integer, Long> sourcePartitionOffsetEnd, //
CubeBuildTypeEnum buildType, boolean force, String submitter, Integer priorityOffset) throws IOException {
Message msg = MsgPicker.getMsg();
if (cube.getStatus() == RealizationStatusEnum.DESCBROKEN) {
throw new BadRequestException(String.format(Locale.ROOT, msg.getBUILD_BROKEN_CUBE(), cube.getName()));
}
checkCubeDescSignature(cube);
checkAllowBuilding(cube);
if (buildType == CubeBuildTypeEnum.BUILD || buildType == CubeBuildTypeEnum.REFRESH) {
checkAllowParallelBuilding(cube);
}
DefaultChainedExecutable job;
CubeSegment newSeg = null;
try {
if (buildType == CubeBuildTypeEnum.BUILD) {
ISource source = SourceManager.getSource(cube);
SourcePartition src = new SourcePartition(tsRange, segRange, sourcePartitionOffsetStart,
sourcePartitionOffsetEnd);
src = source.enrichSourcePartitionBeforeBuild(cube, src);
newSeg = getCubeManager().appendSegment(cube, src);
job = EngineFactory.createBatchCubingJob(newSeg, submitter, priorityOffset);
} else if (buildType == CubeBuildTypeEnum.MERGE) {
newSeg = getCubeManager().mergeSegments(cube, tsRange, segRange, force);
job = EngineFactory.createBatchMergeJob(newSeg, submitter);
} else if (buildType == CubeBuildTypeEnum.REFRESH) {
newSeg = getCubeManager().refreshSegment(cube, tsRange, segRange);
job = EngineFactory.createBatchCubingJob(newSeg, submitter, priorityOffset);
} else {
throw new BadRequestException(String.format(Locale.ROOT, msg.getINVALID_BUILD_TYPE(), buildType));
}
getExecutableManager().addJob(job);
} catch (Exception e) {
if (newSeg != null) {
logger.error("Job submission might failed for NEW segment {}, will clean the NEW segment from cube",
newSeg.getName());
try {
// Remove this segment
getCubeManager().updateCubeDropSegments(cube, newSeg);
} catch (Exception ee) {
// swallow the exception
logger.error("Clean New segment failed, ignoring it", e);
}
}
throw e;
}
JobInstance jobInstance = getSingleJobInstance(job);
return jobInstance;
}
Example #24
| Source File: JobService.java From kylin with Apache License 2.0 | 4 votes |
|
protected JobInstance getSingleJobInstance(AbstractExecutable job) {
Message msg = MsgPicker.getMsg();
if (job == null) {
return null;
}
if (!(job instanceof CubingJob)) {
throw new BadRequestException(String.format(Locale.ROOT, msg.getILLEGAL_JOB_TYPE(), job.getId()));
}
CubingJob cubeJob = (CubingJob) job;
CubeInstance cube = CubeManager.getInstance(KylinConfig.getInstanceFromEnv())
.getCube(CubingExecutableUtil.getCubeName(cubeJob.getParams()));
Output output = cubeJob.getOutput();
final JobInstance result = new JobInstance();
result.setName(job.getName());
result.setProjectName(cubeJob.getProjectName());
if (cube != null) {
result.setRelatedCube(cube.getName());
result.setDisplayCubeName(cube.getDisplayName());
} else {
String cubeName = CubingExecutableUtil.getCubeName(cubeJob.getParams());
result.setRelatedCube(cubeName);
result.setDisplayCubeName(cubeName);
}
result.setRelatedSegment(CubingExecutableUtil.getSegmentId(cubeJob.getParams()));
result.setRelatedSegmentName(CubingExecutableUtil.getSegmentName(cubeJob.getParams()));
result.setLastModified(cubeJob.getLastModified());
result.setSubmitter(cubeJob.getSubmitter());
result.setUuid(cubeJob.getId());
result.setExecStartTime(cubeJob.getStartTime());
result.setExecEndTime(cubeJob.getEndTime());
result.setExecInterruptTime(cubeJob.getInterruptTime());
result.setType(CubeBuildTypeEnum.BUILD);
result.setStatus(JobInfoConverter.parseToJobStatus(job.getStatus()));
result.setMrWaiting(cubeJob.getMapReduceWaitTime() / 1000);
result.setBuildInstance(AbstractExecutable.getBuildInstance(output));
result.setDuration(cubeJob.getDuration() / 1000);
for (int i = 0; i < cubeJob.getTasks().size(); ++i) {
AbstractExecutable task = cubeJob.getTasks().get(i);
result.addStep(JobInfoConverter.parseToJobStep(task, i, getExecutableManager().getOutput(task.getId())));
}
return result;
}
Example #25
| Source File: JobInfoConverter.java From kylin with Apache License 2.0 | 4 votes |
|
public static JobInstance parseToJobInstance(CubingJob job, Map<String, Output> outputs) {
if (job == null) {
logger.warn("job is null.");
return null;
}
Output output = outputs.get(job.getId());
if (output == null) {
logger.warn("job output is null.");
return null;
}
CubingJob cubeJob = job;
String cubeName = CubingExecutableUtil.getCubeName(cubeJob.getParams());
String displayCubeName = cubeName;
try {
CubeInstance cube = CubeManager.getInstance(KylinConfig.getInstanceFromEnv()).getCube(cubeName);
if (cube != null) {
cubeName = cube.getName();
displayCubeName = cube.getDisplayName();
}
} catch (Exception e) {
logger.warn("Fail to get cube instance for {}.", cubeName);
}
final JobInstance result = new JobInstance();
result.setName(job.getName());
result.setProjectName(cubeJob.getProjectName());
result.setRelatedCube(cubeName);
result.setDisplayCubeName(displayCubeName);
result.setRelatedSegment(CubingExecutableUtil.getSegmentId(cubeJob.getParams()));
result.setRelatedSegmentName(CubingExecutableUtil.getSegmentName(cubeJob.getParams()));
result.setLastModified(output.getLastModified());
result.setSubmitter(job.getSubmitter());
result.setUuid(job.getId());
result.setType(CubeBuildTypeEnum.BUILD);
result.setStatus(parseToJobStatus(output.getState()));
result.setBuildInstance(AbstractExecutable.getBuildInstance(output));
result.setMrWaiting(AbstractExecutable.getExtraInfoAsLong(output, CubingJob.MAP_REDUCE_WAIT_TIME, 0L) / 1000);
result.setExecStartTime(AbstractExecutable.getStartTime(output));
result.setExecEndTime(AbstractExecutable.getEndTime(output));
result.setExecInterruptTime(AbstractExecutable.getInterruptTime(output));
result.setDuration(AbstractExecutable.getDuration(result.getExecStartTime(), result.getExecEndTime(),
result.getExecInterruptTime()) / 1000);
for (int i = 0; i < job.getTasks().size(); ++i) {
AbstractExecutable task = job.getTasks().get(i);
result.addStep(parseToJobStep(task, i, outputs.get(task.getId())));
}
return result;
}
Example #26
| Source File: JobInfoConverter.java From kylin-on-parquet-v2 with Apache License 2.0 | 4 votes |
|
public static JobInstance parseToJobInstance(CubingJob job, Map<String, Output> outputs) {
if (job == null) {
logger.warn("job is null.");
return null;
}
Output output = outputs.get(job.getId());
if (output == null) {
logger.warn("job output is null.");
return null;
}
CubingJob cubeJob = (CubingJob) job;
CubeInstance cube = CubeManager.getInstance(KylinConfig.getInstanceFromEnv())
.getCube(CubingExecutableUtil.getCubeName(cubeJob.getParams()));
final JobInstance result = new JobInstance();
result.setName(job.getName());
result.setProjectName(cubeJob.getProjectName());
result.setRelatedCube(cube != null ? cube.getName() : CubingExecutableUtil.getCubeName(cubeJob.getParams()));
result.setDisplayCubeName(cube != null ? cube.getDisplayName() : CubingExecutableUtil.getCubeName(cubeJob.getParams()));
result.setRelatedSegment(CubingExecutableUtil.getSegmentId(cubeJob.getParams()));
result.setRelatedSegmentName(CubingExecutableUtil.getSegmentName(cubeJob.getParams()));
result.setLastModified(output.getLastModified());
result.setSubmitter(job.getSubmitter());
result.setUuid(job.getId());
result.setType(CubeBuildTypeEnum.BUILD);
result.setStatus(parseToJobStatus(output.getState()));
result.setBuildInstance(AbstractExecutable.getBuildInstance(output));
result.setMrWaiting(AbstractExecutable.getExtraInfoAsLong(output, CubingJob.MAP_REDUCE_WAIT_TIME, 0L) / 1000);
result.setExecStartTime(AbstractExecutable.getStartTime(output));
result.setExecEndTime(AbstractExecutable.getEndTime(output));
result.setExecInterruptTime(AbstractExecutable.getInterruptTime(output));
result.setDuration(AbstractExecutable.getDuration(result.getExecStartTime(), result.getExecEndTime(),
result.getExecInterruptTime()) / 1000);
for (int i = 0; i < job.getTasks().size(); ++i) {
AbstractExecutable task = job.getTasks().get(i);
result.addStep(parseToJobStep(task, i, outputs.get(task.getId())));
}
return result;
}
Example #27
| Source File: JobInstance.java From Kylin with Apache License 2.0 | 4 votes |
|
public CubeBuildTypeEnum getType() {
return type;
}
Example #28
| Source File: JobInstance.java From Kylin with Apache License 2.0 | 4 votes |
|
public void setType(CubeBuildTypeEnum type) {
this.type = type;
}
