Java Code Examples for org.apache.kylin.cube.CubeSegment#setLastBuildJobID()
The following examples show how to use
org.apache.kylin.cube.CubeSegment#setLastBuildJobID() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
| Source File: BuildCubeWithEngine.java From kylin-on-parquet-v2 with Apache License 2.0 | 6 votes |
|
private Boolean optimizeCube(String cubeName) throws Exception {
CubeInstance cubeInstance = cubeManager.getCube(cubeName);
Set<Long> cuboidsRecommend = mockRecommendCuboids(cubeInstance, 0.05, 255);
CubeSegment[] optimizeSegments = cubeManager.optimizeSegments(cubeInstance, cuboidsRecommend);
List<AbstractExecutable> optimizeJobList = Lists.newArrayListWithExpectedSize(optimizeSegments.length);
for (CubeSegment optimizeSegment : optimizeSegments) {
DefaultChainedExecutable optimizeJob = EngineFactory.createBatchOptimizeJob(optimizeSegment, "TEST");
jobService.addJob(optimizeJob);
optimizeJobList.add(optimizeJob);
optimizeSegment.setLastBuildJobID(optimizeJob.getId());
}
CheckpointExecutable checkpointJob = new BatchOptimizeJobCheckpointBuilder(cubeInstance, "TEST").build();
checkpointJob.addTaskListForCheck(optimizeJobList);
jobService.addJob(checkpointJob);
if (fastBuildMode) {
return true;
}
ExecutableState state = waitForJob(checkpointJob.getId());
return Boolean.valueOf(ExecutableState.SUCCEED == state);
}
Example 2
| Source File: BuildCubeWithEngine.java From kylin with Apache License 2.0 | 6 votes |
|
private Boolean optimizeCube(String cubeName) throws Exception {
CubeInstance cubeInstance = cubeManager.getCube(cubeName);
Set<Long> cuboidsRecommend = mockRecommendCuboids(cubeInstance, 0.05, 255);
CubeSegment[] optimizeSegments = cubeManager.optimizeSegments(cubeInstance, cuboidsRecommend);
List<AbstractExecutable> optimizeJobList = Lists.newArrayListWithExpectedSize(optimizeSegments.length);
for (CubeSegment optimizeSegment : optimizeSegments) {
DefaultChainedExecutable optimizeJob = EngineFactory.createBatchOptimizeJob(optimizeSegment, "TEST");
jobService.addJob(optimizeJob);
optimizeJobList.add(optimizeJob);
optimizeSegment.setLastBuildJobID(optimizeJob.getId());
}
CheckpointExecutable checkpointJob = new BatchOptimizeJobCheckpointBuilder(cubeInstance, "TEST").build();
checkpointJob.addTaskListForCheck(optimizeJobList);
jobService.addJob(checkpointJob);
if (fastBuildMode) {
return true;
}
ExecutableState state = waitForJob(checkpointJob.getId());
return Boolean.valueOf(ExecutableState.SUCCEED == state);
}
Example 3
| Source File: CubeBuildJob.java From kylin-on-parquet-v2 with Apache License 2.0 | 6 votes |
|
private void updateSegmentInfo(String cubeId, SegmentInfo segmentInfo, long sourceRowCount) throws IOException {
CubeInstance cubeInstance = cubeManager.getCubeByUuid(cubeId);
CubeInstance cubeCopy = cubeInstance.latestCopyForWrite();
CubeUpdate update = new CubeUpdate(cubeCopy);
List<CubeSegment> cubeSegments = Lists.newArrayList();
CubeSegment segment = cubeCopy.getSegmentById(segmentInfo.id());
segment.setSizeKB(segmentInfo.getAllLayoutSize() / 1024);
segment.setLastBuildTime(System.currentTimeMillis());
segment.setLastBuildJobID(getParam(MetadataConstants.P_JOB_ID));
segment.setInputRecords(sourceRowCount);
segment.setSnapshots(new ConcurrentHashMap<>(segmentInfo.getSnapShot2JavaMap()));
segment.setCuboidShardNums(cuboidShardNum);
Map<String, String> additionalInfo = segment.getAdditionalInfo();
additionalInfo.put("storageType", "" + IStorageAware.ID_PARQUET);
segment.setAdditionalInfo(additionalInfo);
cubeSegments.add(segment);
update.setToUpdateSegs(cubeSegments.toArray(new CubeSegment[0]));
cubeManager.updateCube(update);
}
Example 4
| Source File: Coordinator.java From kylin-on-parquet-v2 with Apache License 2.0 | 5 votes |
|
private boolean triggerSegmentBuild(String cubeName, String segmentName) {
CubeManager cubeManager = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
CubeInstance cubeInstance = cubeManager.getCube(cubeName);
try {
Pair<Long, Long> segmentRange = CubeSegment.parseSegmentName(segmentName);
logger.info("submit streaming segment build, cube:{} segment:{}", cubeName, segmentName);
CubeSegment newSeg = getCubeManager().appendSegment(cubeInstance,
new TSRange(segmentRange.getFirst(), segmentRange.getSecond()));
DefaultChainedExecutable executable = new StreamingCubingEngine().createStreamingCubingJob(newSeg,
"SYSTEM");
getExecutableManager().addJob(executable);
CubingJob cubingJob = (CubingJob) executable;
newSeg.setLastBuildJobID(cubingJob.getId());
SegmentJobBuildInfo segmentJobBuildInfo = new SegmentJobBuildInfo(cubeName, segmentName, cubingJob.getId());
jobStatusChecker.addSegmentBuildJob(segmentJobBuildInfo);
SegmentBuildState.BuildState state = new SegmentBuildState.BuildState();
state.setBuildStartTime(System.currentTimeMillis());
state.setState(SegmentBuildState.BuildState.State.BUILDING);
state.setJobId(cubingJob.getId());
streamMetadataStore.updateSegmentBuildState(cubeName, segmentName, state);
return true;
} catch (Exception e) {
logger.error("streaming job submit fail, cubeName:" + cubeName + " segment:" + segmentName, e);
return false;
}
}
Example 5
| Source File: UpdateCubeInfoAfterMergeStep.java From Kylin with Apache License 2.0 | 5 votes |
|
@Override
protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
final CubeInstance cube = cubeManager.getCube(getCubeName());
CubeSegment mergedSegment = cube.getSegmentById(getSegmentId());
if (mergedSegment == null) {
return new ExecuteResult(ExecuteResult.State.FAILED, "there is no segment with id:" + getSegmentId());
}
String cubeSizeString = executableManager.getOutput(getConvertToHfileStepId()).getExtra().get(ExecutableConstants.HDFS_BYTES_WRITTEN);
Preconditions.checkState(StringUtils.isNotEmpty(cubeSizeString), "Can't get cube segment size.");
long cubeSize = Long.parseLong(cubeSizeString) / 1024;
// collect source statistics
List<String> mergingSegmentIds = getMergingSegmentIds();
if (mergingSegmentIds.isEmpty()) {
return new ExecuteResult(ExecuteResult.State.FAILED, "there are no merging segments");
}
long sourceCount = 0L;
long sourceSize = 0L;
for (String id : mergingSegmentIds) {
CubeSegment segment = cube.getSegmentById(id);
sourceCount += segment.getInputRecords();
sourceSize += segment.getInputRecordsSize();
}
// update segment info
mergedSegment.setSizeKB(cubeSize);
mergedSegment.setInputRecords(sourceCount);
mergedSegment.setInputRecordsSize(sourceSize);
mergedSegment.setLastBuildJobID(getCubingJobId());
mergedSegment.setLastBuildTime(System.currentTimeMillis());
try {
cubeManager.promoteNewlyBuiltSegments(cube, mergedSegment);
return new ExecuteResult(ExecuteResult.State.SUCCEED);
} catch (IOException e) {
logger.error("fail to update cube after merge", e);
return new ExecuteResult(ExecuteResult.State.ERROR, e.getLocalizedMessage());
}
}
Example 6
| Source File: JobService.java From kylin with Apache License 2.0 | 5 votes |
|
public void resubmitJob(JobInstance job) throws IOException {
aclEvaluate.checkProjectOperationPermission(job);
Coordinator coordinator = Coordinator.getInstance();
CubeManager cubeManager = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
String cubeName = job.getRelatedCube();
CubeInstance cubeInstance = cubeManager.getCube(cubeName);
String segmentName = job.getRelatedSegmentName();
try {
Pair<Long, Long> segmentRange = CubeSegment.parseSegmentName(segmentName);
logger.info("submit streaming segment build, cube:{} segment:{}", cubeName, segmentName);
CubeSegment newSeg = coordinator.getCubeManager().appendSegment(cubeInstance,
new SegmentRange.TSRange(segmentRange.getFirst(), segmentRange.getSecond()));
DefaultChainedExecutable executable = new StreamingCubingEngine().createStreamingCubingJob(newSeg, aclEvaluate.getCurrentUserName());
coordinator.getExecutableManager().addJob(executable);
CubingJob cubingJob = (CubingJob) executable;
newSeg.setLastBuildJobID(cubingJob.getId());
SegmentBuildState.BuildState state = new SegmentBuildState.BuildState();
state.setBuildStartTime(System.currentTimeMillis());
state.setState(SegmentBuildState.BuildState.State.BUILDING);
state.setJobId(cubingJob.getId());
coordinator.getStreamMetadataStore().updateSegmentBuildState(cubeName, segmentName, state);
} catch (Exception e) {
logger.error("streaming job submit fail, cubeName:" + cubeName + " segment:" + segmentName, e);
throw e;
}
}
Example 7
| Source File: UpdateCubeInfoAfterBuildStep.java From kylin with Apache License 2.0 | 5 votes |
|
@Override
protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
final CubeManager cubeManager = CubeManager.getInstance(context.getConfig());
final CubeInstance cube = cubeManager.getCube(CubingExecutableUtil.getCubeName(this.getParams()))
.latestCopyForWrite();
final CubeSegment segment = cube.getSegmentById(CubingExecutableUtil.getSegmentId(this.getParams()));
CubingJob cubingJob = (CubingJob) getManager().getJob(CubingExecutableUtil.getCubingJobId(this.getParams()));
long sourceCount = cubingJob.findSourceRecordCount();
long sourceSizeBytes = cubingJob.findSourceSizeBytes();
long cubeSizeBytes = cubingJob.findCubeSizeBytes();
KylinConfig config = KylinConfig.getInstanceFromEnv();
List<Double> cuboidEstimateRatio = cubingJob.findEstimateRatio(segment, config);
segment.setLastBuildJobID(CubingExecutableUtil.getCubingJobId(this.getParams()));
segment.setLastBuildTime(System.currentTimeMillis());
segment.setSizeKB(cubeSizeBytes / 1024);
segment.setInputRecords(sourceCount);
segment.setInputRecordsSize(sourceSizeBytes);
segment.setEstimateRatio(cuboidEstimateRatio);
try {
deleteDictionaryIfNeeded(segment);
saveExtSnapshotIfNeeded(cubeManager, cube, segment);
updateSegment(segment);
cubeManager.promoteNewlyBuiltSegments(cube, segment);
return new ExecuteResult();
} catch (IOException e) {
logger.error("fail to update cube after build", e);
return ExecuteResult.createError(e);
}
}
Example 8
| Source File: UpdateCubeInfoAfterOptimizeStep.java From kylin with Apache License 2.0 | 5 votes |
|
@Override
protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
final CubeManager cubeManager = CubeManager.getInstance(context.getConfig());
final CubeInstance cube = cubeManager.getCube(CubingExecutableUtil.getCubeName(this.getParams()));
final CubeSegment segment = cube.getSegmentById(CubingExecutableUtil.getSegmentId(this.getParams()));
CubeSegment originalSegment = cube.getOriginalSegmentToOptimize(segment);
long sourceCount = originalSegment.getInputRecords();
long sourceSizeBytes = originalSegment.getInputRecordsSize();
CubingJob cubingJob = (CubingJob) getManager().getJob(CubingExecutableUtil.getCubingJobId(this.getParams()));
long cubeSizeBytes = cubingJob.findCubeSizeBytes();
segment.setLastBuildJobID(CubingExecutableUtil.getCubingJobId(this.getParams()));
segment.setLastBuildTime(System.currentTimeMillis());
segment.setSizeKB(cubeSizeBytes / 1024);
segment.setInputRecords(sourceCount);
segment.setInputRecordsSize(sourceSizeBytes);
segment.setDimensionRangeInfoMap(originalSegment.getDimensionRangeInfoMap());
try {
cubeManager.promoteNewlyOptimizeSegments(cube, segment);
return new ExecuteResult();
} catch (IOException e) {
logger.error("fail to update cube after build", e);
return ExecuteResult.createError(e);
}
}
Example 9
| Source File: ReceiverClusterManager.java From kylin with Apache License 2.0 | 5 votes |
|
/**
* Promote a segment from realtime part into historical part.
*/
void promoteNewSegment(CubingJob cubingJob, CubeInstance cubeInstance, CubeSegment cubeSegment) throws IOException {
logger.debug("Try transfer segment's {} state to ready.", cubeSegment.getName());
long sourceCount = cubingJob.findSourceRecordCount();
long sourceSizeBytes = cubingJob.findSourceSizeBytes();
long cubeSizeBytes = cubingJob.findCubeSizeBytes();
Map<Integer, String> sourceCheckpoint = getCoordinator().getStreamMetadataStore()
.getSourceCheckpoint(cubeInstance.getName(), cubeSegment.getName());
ISourcePositionHandler positionOperator = StreamingSourceFactory.getStreamingSource(cubeInstance)
.getSourcePositionHandler();
Collection<ISourcePosition> sourcePositions = Collections2.transform(sourceCheckpoint.values(),
new Function<String, ISourcePosition>() {
@Nullable
@Override
public ISourcePosition apply(@Nullable String input) {
return positionOperator.parsePosition(input);
}
});
ISourcePosition sourcePosition = positionOperator.mergePositions(sourcePositions,
ISourcePositionHandler.MergeStrategy.KEEP_SMALL);
cubeSegment.setLastBuildJobID(cubingJob.getId());
cubeSegment.setLastBuildTime(System.currentTimeMillis());
cubeSegment.setSizeKB(cubeSizeBytes / 1024);
cubeSegment.setInputRecords(sourceCount);
cubeSegment.setInputRecordsSize(sourceSizeBytes);
cubeSegment.setStreamSourceCheckpoint(positionOperator.serializePosition(sourcePosition));
getCoordinator().getCubeManager().promoteNewlyBuiltSegments(cubeInstance, cubeSegment);
}
Example 10
| Source File: Coordinator.java From kylin with Apache License 2.0 | 5 votes |
|
private boolean triggerSegmentBuild(String cubeName, String segmentName) {
CubeManager cubeManager = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
CubeInstance cubeInstance = cubeManager.getCube(cubeName);
try {
Pair<Long, Long> segmentRange = CubeSegment.parseSegmentName(segmentName);
logger.info("submit streaming segment build, cube:{} segment:{}", cubeName, segmentName);
CubeSegment newSeg = getCubeManager().appendSegment(cubeInstance,
new TSRange(segmentRange.getFirst(), segmentRange.getSecond()));
DefaultChainedExecutable executable = new StreamingCubingEngine().createStreamingCubingJob(newSeg,
"SYSTEM");
getExecutableManager().addJob(executable);
CubingJob cubingJob = (CubingJob) executable;
newSeg.setLastBuildJobID(cubingJob.getId());
SegmentJobBuildInfo segmentJobBuildInfo = new SegmentJobBuildInfo(cubeName, segmentName, cubingJob.getId());
jobStatusChecker.addSegmentBuildJob(segmentJobBuildInfo);
SegmentBuildState.BuildState state = new SegmentBuildState.BuildState();
state.setBuildStartTime(System.currentTimeMillis());
state.setState(SegmentBuildState.BuildState.State.BUILDING);
state.setJobId(cubingJob.getId());
streamMetadataStore.updateSegmentBuildState(cubeName, segmentName, state);
return true;
} catch (Exception e) {
logger.error("streaming job submit fail, cubeName:" + cubeName + " segment:" + segmentName, e);
return false;
}
}
Example 11
| Source File: Coordinator.java From kylin with Apache License 2.0 | 5 votes |
|
private void promoteNewSegment(CubingJob cubingJob, CubeInstance cubeInstance, CubeSegment cubeSegment)
throws IOException {
long sourceCount = cubingJob.findSourceRecordCount();
long sourceSizeBytes = cubingJob.findSourceSizeBytes();
long cubeSizeBytes = cubingJob.findCubeSizeBytes();
Map<Integer, String> sourceCheckpoint = streamMetadataStore.getSourceCheckpoint(cubeInstance.getName(),
cubeSegment.getName());
ISourcePositionHandler positionOperator = StreamingSourceFactory.getStreamingSource(cubeInstance)
.getSourcePositionHandler();
Collection<ISourcePosition> sourcePositions = Collections2.transform(sourceCheckpoint.values(),
new Function<String, ISourcePosition>() {
@Nullable
@Override
public ISourcePosition apply(@Nullable String input) {
return positionOperator.parsePosition(input);
}
});
ISourcePosition sourcePosition = positionOperator.mergePositions(sourcePositions, MergeStrategy.KEEP_SMALL);
cubeSegment.setLastBuildJobID(cubingJob.getId());
cubeSegment.setLastBuildTime(System.currentTimeMillis());
cubeSegment.setSizeKB(cubeSizeBytes / 1024);
cubeSegment.setInputRecords(sourceCount);
cubeSegment.setInputRecordsSize(sourceSizeBytes);
cubeSegment.setStreamSourceCheckpoint(positionOperator.serializePosition(sourcePosition));
getCubeManager().promoteNewlyBuiltSegments(cubeInstance, cubeSegment);
}
Example 12
| Source File: JobService.java From kylin-on-parquet-v2 with Apache License 2.0 | 5 votes |
|
public void resubmitJob(JobInstance job) throws IOException {
aclEvaluate.checkProjectOperationPermission(job);
Coordinator coordinator = Coordinator.getInstance();
CubeManager cubeManager = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
String cubeName = job.getRelatedCube();
CubeInstance cubeInstance = cubeManager.getCube(cubeName);
String segmentName = job.getRelatedSegmentName();
try {
Pair<Long, Long> segmentRange = CubeSegment.parseSegmentName(segmentName);
logger.info("submit streaming segment build, cube:{} segment:{}", cubeName, segmentName);
CubeSegment newSeg = coordinator.getCubeManager().appendSegment(cubeInstance,
new SegmentRange.TSRange(segmentRange.getFirst(), segmentRange.getSecond()));
DefaultChainedExecutable executable = new StreamingCubingEngine().createStreamingCubingJob(newSeg, aclEvaluate.getCurrentUserName());
coordinator.getExecutableManager().addJob(executable);
CubingJob cubingJob = (CubingJob) executable;
newSeg.setLastBuildJobID(cubingJob.getId());
SegmentBuildState.BuildState state = new SegmentBuildState.BuildState();
state.setBuildStartTime(System.currentTimeMillis());
state.setState(SegmentBuildState.BuildState.State.BUILDING);
state.setJobId(cubingJob.getId());
coordinator.getStreamMetadataStore().updateSegmentBuildState(cubeName, segmentName, state);
} catch (Exception e) {
logger.error("streaming job submit fail, cubeName:" + cubeName + " segment:" + segmentName, e);
throw e;
}
}
Example 13
| Source File: UpdateCubeInfoAfterBuildStep.java From kylin-on-parquet-v2 with Apache License 2.0 | 5 votes |
|
@Override
protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
final CubeManager cubeManager = CubeManager.getInstance(context.getConfig());
final CubeInstance cube = cubeManager.getCube(CubingExecutableUtil.getCubeName(this.getParams()))
.latestCopyForWrite();
final CubeSegment segment = cube.getSegmentById(CubingExecutableUtil.getSegmentId(this.getParams()));
CubingJob cubingJob = (CubingJob) getManager().getJob(CubingExecutableUtil.getCubingJobId(this.getParams()));
long sourceCount = cubingJob.findSourceRecordCount();
long sourceSizeBytes = cubingJob.findSourceSizeBytes();
long cubeSizeBytes = cubingJob.findCubeSizeBytes();
KylinConfig config = KylinConfig.getInstanceFromEnv();
List<Double> cuboidEstimateRatio = cubingJob.findEstimateRatio(segment, config);
segment.setLastBuildJobID(CubingExecutableUtil.getCubingJobId(this.getParams()));
segment.setLastBuildTime(System.currentTimeMillis());
segment.setSizeKB(cubeSizeBytes / 1024);
segment.setInputRecords(sourceCount);
segment.setInputRecordsSize(sourceSizeBytes);
segment.setEstimateRatio(cuboidEstimateRatio);
try {
saveExtSnapshotIfNeeded(cubeManager, cube, segment);
updateSegment(segment);
cubeManager.promoteNewlyBuiltSegments(cube, segment);
return new ExecuteResult();
} catch (IOException e) {
logger.error("fail to update cube after build", e);
return ExecuteResult.createError(e);
}
}
Example 14
| Source File: UpdateCubeInfoAfterOptimizeStep.java From kylin-on-parquet-v2 with Apache License 2.0 | 5 votes |
|
@Override
protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
final CubeManager cubeManager = CubeManager.getInstance(context.getConfig());
final CubeInstance cube = cubeManager.getCube(CubingExecutableUtil.getCubeName(this.getParams()));
final CubeSegment segment = cube.getSegmentById(CubingExecutableUtil.getSegmentId(this.getParams()));
CubeSegment originalSegment = cube.getOriginalSegmentToOptimize(segment);
long sourceCount = originalSegment.getInputRecords();
long sourceSizeBytes = originalSegment.getInputRecordsSize();
CubingJob cubingJob = (CubingJob) getManager().getJob(CubingExecutableUtil.getCubingJobId(this.getParams()));
long cubeSizeBytes = cubingJob.findCubeSizeBytes();
segment.setLastBuildJobID(CubingExecutableUtil.getCubingJobId(this.getParams()));
segment.setLastBuildTime(System.currentTimeMillis());
segment.setSizeKB(cubeSizeBytes / 1024);
segment.setInputRecords(sourceCount);
segment.setInputRecordsSize(sourceSizeBytes);
segment.setDimensionRangeInfoMap(originalSegment.getDimensionRangeInfoMap());
try {
cubeManager.promoteNewlyOptimizeSegments(cube, segment);
return new ExecuteResult();
} catch (IOException e) {
logger.error("fail to update cube after build", e);
return ExecuteResult.createError(e);
}
}
Example 15
| Source File: ReceiverClusterManager.java From kylin-on-parquet-v2 with Apache License 2.0 | 5 votes |
|
/**
* Promote a segment from realtime part into historical part.
*/
void promoteNewSegment(CubingJob cubingJob, CubeInstance cubeInstance, CubeSegment cubeSegment) throws IOException {
logger.debug("Try transfer segment's {} state to ready.", cubeSegment.getName());
long sourceCount = cubingJob.findSourceRecordCount();
long sourceSizeBytes = cubingJob.findSourceSizeBytes();
long cubeSizeBytes = cubingJob.findCubeSizeBytes();
Map<Integer, String> sourceCheckpoint = getCoordinator().getStreamMetadataStore()
.getSourceCheckpoint(cubeInstance.getName(), cubeSegment.getName());
ISourcePositionHandler positionOperator = StreamingSourceFactory.getStreamingSource(cubeInstance)
.getSourcePositionHandler();
Collection<ISourcePosition> sourcePositions = Collections2.transform(sourceCheckpoint.values(),
new Function<String, ISourcePosition>() {
@Nullable
@Override
public ISourcePosition apply(@Nullable String input) {
return positionOperator.parsePosition(input);
}
});
ISourcePosition sourcePosition = positionOperator.mergePositions(sourcePositions,
ISourcePositionHandler.MergeStrategy.KEEP_SMALL);
cubeSegment.setLastBuildJobID(cubingJob.getId());
cubeSegment.setLastBuildTime(System.currentTimeMillis());
cubeSegment.setSizeKB(cubeSizeBytes / 1024);
cubeSegment.setInputRecords(sourceCount);
cubeSegment.setInputRecordsSize(sourceSizeBytes);
cubeSegment.setStreamSourceCheckpoint(positionOperator.serializePosition(sourcePosition));
getCoordinator().getCubeManager().promoteNewlyBuiltSegments(cubeInstance, cubeSegment);
}
Example 16
| Source File: Coordinator.java From kylin-on-parquet-v2 with Apache License 2.0 | 5 votes |
|
private void promoteNewSegment(CubingJob cubingJob, CubeInstance cubeInstance, CubeSegment cubeSegment)
throws IOException {
long sourceCount = cubingJob.findSourceRecordCount();
long sourceSizeBytes = cubingJob.findSourceSizeBytes();
long cubeSizeBytes = cubingJob.findCubeSizeBytes();
Map<Integer, String> sourceCheckpoint = streamMetadataStore.getSourceCheckpoint(cubeInstance.getName(),
cubeSegment.getName());
ISourcePositionHandler positionOperator = StreamingSourceFactory.getStreamingSource(cubeInstance)
.getSourcePositionHandler();
Collection<ISourcePosition> sourcePositions = Collections2.transform(sourceCheckpoint.values(),
new Function<String, ISourcePosition>() {
@Nullable
@Override
public ISourcePosition apply(@Nullable String input) {
return positionOperator.parsePosition(input);
}
});
ISourcePosition sourcePosition = positionOperator.mergePositions(sourcePositions, MergeStrategy.KEEP_SMALL);
cubeSegment.setLastBuildJobID(cubingJob.getId());
cubeSegment.setLastBuildTime(System.currentTimeMillis());
cubeSegment.setSizeKB(cubeSizeBytes / 1024);
cubeSegment.setInputRecords(sourceCount);
cubeSegment.setInputRecordsSize(sourceSizeBytes);
cubeSegment.setStreamSourceCheckpoint(positionOperator.serializePosition(sourcePosition));
getCubeManager().promoteNewlyBuiltSegments(cubeInstance, cubeSegment);
}
Example 17
| Source File: BuildJobSubmitter.java From kylin with Apache License 2.0 | 4 votes |
|
/**
* Submit a build job for streaming segment
*
* @return true if submit succeed ; else false
*/
@NotAtomicIdempotent
boolean submitSegmentBuildJob(String cubeName, String segmentName) {
logger.info("Try submit streaming segment build job, cube:{} segment:{}", cubeName, segmentName);
CubeInstance cubeInstance = coordinator.getCubeManager().getCube(cubeName);
try {
// Step 1. create a new segment if not exists
CubeSegment newSeg = null;
Pair<Long, Long> segmentRange = CubeSegment.parseSegmentName(segmentName);
boolean segmentExists = false;
for (CubeSegment segment : cubeInstance.getSegments()) {
SegmentRange.TSRange tsRange = segment.getTSRange();
if (tsRange.start.v.equals(segmentRange.getFirst()) && segmentRange.getSecond().equals(tsRange.end.v)) {
segmentExists = true;
newSeg = segment;
}
}
if (segmentExists) {
logger.warn("Segment {} exists, it will be forced deleted.", segmentName);
coordinator.getCubeManager().updateCubeDropSegments(cubeInstance, newSeg);
}
logger.debug("Create segment for {} {} .", cubeName, segmentName);
newSeg = coordinator.getCubeManager().appendSegment(cubeInstance,
new SegmentRange.TSRange(segmentRange.getFirst(), segmentRange.getSecond()));
// Step 2. create and submit new build job
DefaultChainedExecutable executable = getStreamingCubingJob(newSeg);
coordinator.getExecutableManager().addJob(executable);
String jobId = executable.getId();
newSeg.setLastBuildJobID(jobId);
// Step 3. add it to job trigger list
SegmentJobBuildInfo segmentJobBuildInfo = new SegmentJobBuildInfo(cubeName, segmentName, jobId);
addToJobTrackList(segmentJobBuildInfo);
// Step 4. add job to stream metadata in case of current node dead
SegmentBuildState.BuildState state = new SegmentBuildState.BuildState();
state.setBuildStartTime(System.currentTimeMillis());
state.setState(SegmentBuildState.BuildState.State.BUILDING);
state.setJobId(jobId);
logger.debug("Commit building job {} for {} {} .", jobId, cubeName, segmentName);
coordinator.getStreamMetadataStore().updateSegmentBuildState(cubeName, segmentName, state);
return true;
} catch (Exception e) {
logger.error("Streaming job submit fail, cubeName:" + cubeName + " segment:" + segmentName, e);
return false;
}
}
Example 18
| Source File: BuildJobSubmitter.java From kylin-on-parquet-v2 with Apache License 2.0 | 4 votes |
|
/**
* Submit a build job for streaming segment
*
* @return true if submit succeed ; else false
*/
@NotAtomicIdempotent
boolean submitSegmentBuildJob(String cubeName, String segmentName) {
logger.info("Try submit streaming segment build job, cube:{} segment:{}", cubeName, segmentName);
CubeInstance cubeInstance = coordinator.getCubeManager().getCube(cubeName);
try {
// Step 1. create a new segment if not exists
CubeSegment newSeg = null;
Pair<Long, Long> segmentRange = CubeSegment.parseSegmentName(segmentName);
boolean segmentExists = false;
for (CubeSegment segment : cubeInstance.getSegments()) {
SegmentRange.TSRange tsRange = segment.getTSRange();
if (tsRange.start.v.equals(segmentRange.getFirst()) && segmentRange.getSecond().equals(tsRange.end.v)) {
segmentExists = true;
newSeg = segment;
}
}
if (segmentExists) {
logger.warn("Segment {} exists, it will be forced deleted.", segmentName);
coordinator.getCubeManager().updateCubeDropSegments(cubeInstance, newSeg);
}
logger.debug("Create segment for {} {} .", cubeName, segmentName);
newSeg = coordinator.getCubeManager().appendSegment(cubeInstance,
new SegmentRange.TSRange(segmentRange.getFirst(), segmentRange.getSecond()));
// Step 2. create and submit new build job
DefaultChainedExecutable executable = getStreamingCubingJob(newSeg);
coordinator.getExecutableManager().addJob(executable);
String jobId = executable.getId();
newSeg.setLastBuildJobID(jobId);
// Step 3. add it to job trigger list
SegmentJobBuildInfo segmentJobBuildInfo = new SegmentJobBuildInfo(cubeName, segmentName, jobId);
addToJobTrackList(segmentJobBuildInfo);
// Step 4. add job to stream metadata in case of current node dead
SegmentBuildState.BuildState state = new SegmentBuildState.BuildState();
state.setBuildStartTime(System.currentTimeMillis());
state.setState(SegmentBuildState.BuildState.State.BUILDING);
state.setJobId(jobId);
logger.debug("Commit building job {} for {} {} .", jobId, cubeName, segmentName);
coordinator.getStreamMetadataStore().updateSegmentBuildState(cubeName, segmentName, state);
return true;
} catch (Exception e) {
logger.error("Streaming job submit fail, cubeName:" + cubeName + " segment:" + segmentName, e);
return false;
}
}
Example 19
| Source File: UpdateCubeInfoAfterBuildStep.java From Kylin with Apache License 2.0 | 4 votes |
|
@Override
protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
final CubeManager cubeManager = CubeManager.getInstance(context.getConfig());
final CubeInstance cube = cubeManager.getCube(getCubeName());
final CubeSegment segment = cube.getSegmentById(getSegmentId());
Output baseCuboidOutput = executableManager.getOutput(getBaseCuboidStepId());
String sourceRecordsCount = baseCuboidOutput.getExtra().get(ExecutableConstants.SOURCE_RECORDS_COUNT);
Preconditions.checkState(StringUtils.isNotEmpty(sourceRecordsCount), "Can't get cube source record count.");
long sourceCount = Long.parseLong(sourceRecordsCount);
String sourceRecordsSize = baseCuboidOutput.getExtra().get(ExecutableConstants.SOURCE_RECORDS_SIZE);
Preconditions.checkState(StringUtils.isNotEmpty(sourceRecordsSize), "Can't get cube source record size.");
long sourceSize = Long.parseLong(sourceRecordsSize);
long size = 0;
boolean segmentReady = true;
if (!StringUtils.isBlank(getConvertToHfileStepId())) {
String cubeSizeString = executableManager.getOutput(getConvertToHfileStepId()).getExtra().get(ExecutableConstants.HDFS_BYTES_WRITTEN);
Preconditions.checkState(StringUtils.isNotEmpty(cubeSizeString), "Can't get cube segment size.");
size = Long.parseLong(cubeSizeString) / 1024;
} else {
// for the increment & merge case, the increment segment is only built to be merged, won't serve query by itself
segmentReady = false;
}
segment.setLastBuildJobID(getCubingJobId());
segment.setLastBuildTime(System.currentTimeMillis());
segment.setSizeKB(size);
segment.setInputRecords(sourceCount);
segment.setInputRecordsSize(sourceSize);
try {
if (segmentReady) {
cubeManager.promoteNewlyBuiltSegments(cube, segment);
} else {
cubeManager.updateCube(cube);
}
return new ExecuteResult(ExecuteResult.State.SUCCEED, "succeed");
} catch (IOException e) {
logger.error("fail to update cube after build", e);
return new ExecuteResult(ExecuteResult.State.ERROR, e.getLocalizedMessage());
}
}
