Java Code Examples for org.apache.kylin.cube.CubeInstance#latestCopyForWrite()
The following examples show how to use
org.apache.kylin.cube.CubeInstance#latestCopyForWrite() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
| Source File: StatisticsDecisionUtil.java From kylin-on-parquet-v2 with Apache License 2.0 | 6 votes |
|
public static void optimizeCubingPlan(CubeSegment segment) throws IOException {
if (isAbleToOptimizeCubingPlan(segment)) {
logger.info("It's able to trigger cuboid planner algorithm.");
} else {
return;
}
Map<Long, Long> recommendCuboidsWithStats = CuboidRecommenderUtil.getRecommendCuboidList(segment);
if (recommendCuboidsWithStats == null || recommendCuboidsWithStats.isEmpty()) {
return;
}
CubeInstance cube = segment.getCubeInstance();
CubeUpdate update = new CubeUpdate(cube.latestCopyForWrite());
update.setCuboids(recommendCuboidsWithStats);
CubeManager.getInstance(cube.getConfig()).updateCube(update);
}
Example 2
| Source File: SparkExecutable.java From kylin-on-parquet-v2 with Apache License 2.0 | 6 votes |
|
private void updateSparkDimensionDicMetadata(KylinConfig config, CubeInstance cube, String segmentId)
throws IOException {
KylinConfig hdfsConfig = AbstractHadoopJob
.loadKylinConfigFromHdfs(this.getParam(SparkBuildDictionary.OPTION_META_URL.getOpt()));
CubeInstance cubeInstance = CubeManager.getInstance(hdfsConfig).reloadCube(cube.getName());
CubeSegment segment = cubeInstance.getSegmentById(segmentId);
CubeSegment oldSeg = cube.getSegmentById(segmentId);
oldSeg.setDictionaries((ConcurrentHashMap<String, String>) segment.getDictionaries());
oldSeg.setSnapshots((ConcurrentHashMap) segment.getSnapshots());
oldSeg.getRowkeyStats().addAll(segment.getRowkeyStats());
CubeInstance cubeCopy = cube.latestCopyForWrite();
CubeUpdate update = new CubeUpdate(cubeCopy);
update.setToUpdateSegs(oldSeg);
CubeManager.getInstance(config).updateCube(update);
Set<String> dumpList = new LinkedHashSet<>();
dumpList.addAll(segment.getDictionaryPaths());
dumpList.addAll(segment.getSnapshotPaths());
JobRelatedMetaUtil.dumpAndUploadKylinPropsAndMetadata(dumpList, (KylinConfigExt) segment.getConfig(),
config.getMetadataUrl().toString());
}
Example 3
| Source File: CubeBuildJob.java From kylin-on-parquet-v2 with Apache License 2.0 | 6 votes |
|
private void updateSegmentInfo(String cubeId, SegmentInfo segmentInfo, long sourceRowCount) throws IOException {
CubeInstance cubeInstance = cubeManager.getCubeByUuid(cubeId);
CubeInstance cubeCopy = cubeInstance.latestCopyForWrite();
CubeUpdate update = new CubeUpdate(cubeCopy);
List<CubeSegment> cubeSegments = Lists.newArrayList();
CubeSegment segment = cubeCopy.getSegmentById(segmentInfo.id());
segment.setSizeKB(segmentInfo.getAllLayoutSize() / 1024);
segment.setLastBuildTime(System.currentTimeMillis());
segment.setLastBuildJobID(getParam(MetadataConstants.P_JOB_ID));
segment.setInputRecords(sourceRowCount);
segment.setSnapshots(new ConcurrentHashMap<>(segmentInfo.getSnapShot2JavaMap()));
segment.setCuboidShardNums(cuboidShardNum);
Map<String, String> additionalInfo = segment.getAdditionalInfo();
additionalInfo.put("storageType", "" + IStorageAware.ID_PARQUET);
segment.setAdditionalInfo(additionalInfo);
cubeSegments.add(segment);
update.setToUpdateSegs(cubeSegments.toArray(new CubeSegment[0]));
cubeManager.updateCube(update);
}
Example 4
| Source File: StatisticsDecisionUtil.java From kylin with Apache License 2.0 | 6 votes |
|
public static void optimizeCubingPlan(CubeSegment segment) throws IOException {
if (isAbleToOptimizeCubingPlan(segment)) {
logger.info("It's able to trigger cuboid planner algorithm.");
} else {
return;
}
Map<Long, Long> recommendCuboidsWithStats = CuboidRecommenderUtil.getRecommendCuboidList(segment);
if (recommendCuboidsWithStats == null || recommendCuboidsWithStats.isEmpty()) {
return;
}
CubeInstance cube = segment.getCubeInstance();
CubeUpdate update = new CubeUpdate(cube.latestCopyForWrite());
update.setCuboids(recommendCuboidsWithStats);
CubeManager.getInstance(cube.getConfig()).updateCube(update);
}
Example 5
| Source File: SparkExecutable.java From kylin with Apache License 2.0 | 6 votes |
|
private void updateSparkDimensionDicMetadata(KylinConfig config, CubeInstance cube, String segmentId)
throws IOException {
KylinConfig hdfsConfig = AbstractHadoopJob
.loadKylinConfigFromHdfs(this.getParam(SparkBuildDictionary.OPTION_META_URL.getOpt()));
CubeInstance cubeInstance = CubeManager.getInstance(hdfsConfig).reloadCube(cube.getName());
CubeSegment segment = cubeInstance.getSegmentById(segmentId);
CubeSegment oldSeg = cube.getSegmentById(segmentId);
oldSeg.setDictionaries((ConcurrentHashMap<String, String>) segment.getDictionaries());
oldSeg.setSnapshots((ConcurrentHashMap) segment.getSnapshots());
oldSeg.getRowkeyStats().addAll(segment.getRowkeyStats());
CubeInstance cubeCopy = cube.latestCopyForWrite();
CubeUpdate update = new CubeUpdate(cubeCopy);
update.setToUpdateSegs(oldSeg);
CubeManager.getInstance(config).updateCube(update);
Set<String> dumpList = new LinkedHashSet<>();
dumpList.addAll(segment.getDictionaryPaths());
dumpList.addAll(segment.getSnapshotPaths());
JobRelatedMetaUtil.dumpAndUploadKylinPropsAndMetadata(dumpList, (KylinConfigExt) segment.getConfig(),
config.getMetadataUrl().toString());
}
Example 6
| Source File: MergeDictionaryStep.java From kylin-on-parquet-v2 with Apache License 2.0 | 5 votes |
|
@Override
protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
final CubeManager mgr = CubeManager.getInstance(context.getConfig());
final CubeInstance cube = mgr.getCube(CubingExecutableUtil.getCubeName(this.getParams()));
final CubeSegment newSegment = cube.getSegmentById(CubingExecutableUtil.getSegmentId(this.getParams()));
final List<CubeSegment> mergingSegments = getMergingSegments(cube);
KylinConfig conf = cube.getConfig();
Collections.sort(mergingSegments);
try {
checkLookupSnapshotsMustIncremental(mergingSegments);
// work on copy instead of cached objects
CubeInstance cubeCopy = cube.latestCopyForWrite();
CubeSegment newSegCopy = cubeCopy.getSegmentById(newSegment.getUuid());
makeDictForNewSegment(conf, cubeCopy, newSegCopy, mergingSegments);
makeSnapshotForNewSegment(cubeCopy, newSegCopy, mergingSegments);
CubeUpdate update = new CubeUpdate(cubeCopy);
update.setToUpdateSegs(newSegCopy);
mgr.updateCube(update);
return ExecuteResult.createSucceed();
} catch (IOException e) {
logger.error("fail to merge dictionary or lookup snapshots", e);
return ExecuteResult.createError(e);
}
}
Example 7
| Source File: CheckpointExecutable.java From kylin-on-parquet-v2 with Apache License 2.0 | 5 votes |
|
@Override
protected void onExecuteFinished(ExecuteResult result, ExecutableContext executableContext) {
super.onExecuteFinished(result, executableContext);
if (!isDiscarded() && result.succeed()) {
List<? extends Executable> jobs = getTasks();
boolean allSucceed = true;
for (Executable task : jobs) {
final ExecutableState status = task.getStatus();
if (status != ExecutableState.SUCCEED) {
allSucceed = false;
}
}
if (allSucceed) {
// Add last optimization time
CubeManager cubeManager = CubeManager.getInstance(executableContext.getConfig());
CubeInstance cube = cubeManager.getCube(getCubeName());
CubeInstance copyForWrite = cube.latestCopyForWrite();
try {
copyForWrite.setCuboidLastOptimized(getEndTime());
CubeUpdate cubeUpdate = new CubeUpdate(copyForWrite);
cubeManager.updateCube(cubeUpdate);
} catch (IOException e) {
logger.error("Failed to update last optimized for " + getCubeName(), e);
}
}
}
}
Example 8
| Source File: SignatureCalculatorTest.java From kylin-on-parquet-v2 with Apache License 2.0 | 5 votes |
|
private CubeInstance cloneCubeInstance(CubeManager cubeManager, CubeInstance cube, String name) throws IOException {
CubeInstance cubeClone = cubeManager.createCube(name, projectName, cube.getDescriptor(), cube.getOwner());
CubeUpdate cubeUpdate = new CubeUpdate(cubeClone.latestCopyForWrite());
cubeUpdate.setToAddSegs(cube.getSegments().toArray(new CubeSegment[cube.getSegments().size()]));
cubeUpdate.setStatus(RealizationStatusEnum.READY);
return cubeManager.updateCube(cubeUpdate);
}
Example 9
| Source File: CubeBuildJob.java From kylin-on-parquet-v2 with Apache License 2.0 | 5 votes |
|
private void updateSegmentSourceBytesSize(String cubeId, Map<String, Object> toUpdateSegmentSourceSize)
throws IOException {
CubeInstance cubeInstance = cubeManager.getCubeByUuid(cubeId);
CubeInstance cubeCopy = cubeInstance.latestCopyForWrite();
CubeUpdate update = new CubeUpdate(cubeCopy);
List<CubeSegment> cubeSegments = Lists.newArrayList();
for (Map.Entry<String, Object> entry : toUpdateSegmentSourceSize.entrySet()) {
CubeSegment segment = cubeCopy.getSegmentById(entry.getKey());
segment.setInputRecordsSize((Long) entry.getValue());
segment.setLastBuildTime(System.currentTimeMillis());
cubeSegments.add(segment);
}
update.setToUpdateSegs(cubeSegments.toArray(new CubeSegment[0]));
cubeManager.updateCube(update);
}
Example 10
| Source File: MergeDictionaryStep.java From kylin with Apache License 2.0 | 5 votes |
|
@Override
protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
final CubeManager mgr = CubeManager.getInstance(context.getConfig());
final CubeInstance cube = mgr.getCube(CubingExecutableUtil.getCubeName(this.getParams()));
final CubeSegment newSegment = cube.getSegmentById(CubingExecutableUtil.getSegmentId(this.getParams()));
final List<CubeSegment> mergingSegments = getMergingSegments(cube);
KylinConfig conf = cube.getConfig();
Collections.sort(mergingSegments);
try {
checkLookupSnapshotsMustIncremental(mergingSegments);
// work on copy instead of cached objects
CubeInstance cubeCopy = cube.latestCopyForWrite();
CubeSegment newSegCopy = cubeCopy.getSegmentById(newSegment.getUuid());
makeDictForNewSegment(conf, cubeCopy, newSegCopy, mergingSegments);
makeSnapshotForNewSegment(cubeCopy, newSegCopy, mergingSegments);
CubeUpdate update = new CubeUpdate(cubeCopy);
update.setToUpdateSegs(newSegCopy);
mgr.updateCube(update);
return ExecuteResult.createSucceed();
} catch (IOException e) {
logger.error("fail to merge dictionary or lookup snapshots", e);
return ExecuteResult.createError(e);
}
}
Example 11
| Source File: CheckpointExecutable.java From kylin with Apache License 2.0 | 5 votes |
|
@Override
protected void onExecuteFinished(ExecuteResult result, ExecutableContext executableContext) {
super.onExecuteFinished(result, executableContext);
if (!isDiscarded() && result.succeed()) {
List<? extends Executable> jobs = getTasks();
boolean allSucceed = true;
for (Executable task : jobs) {
final ExecutableState status = task.getStatus();
if (status != ExecutableState.SUCCEED) {
allSucceed = false;
}
}
if (allSucceed) {
// Add last optimization time
CubeManager cubeManager = CubeManager.getInstance(executableContext.getConfig());
CubeInstance cube = cubeManager.getCube(getCubeName());
CubeInstance copyForWrite = cube.latestCopyForWrite();
try {
copyForWrite.setCuboidLastOptimized(getEndTime());
CubeUpdate cubeUpdate = new CubeUpdate(copyForWrite);
cubeManager.updateCube(cubeUpdate);
} catch (IOException e) {
logger.error("Failed to update last optimized for " + getCubeName(), e);
}
}
}
}
Example 12
| Source File: SignatureCalculatorTest.java From kylin with Apache License 2.0 | 5 votes |
|
private CubeInstance cloneCubeInstance(CubeManager cubeManager, CubeInstance cube, String name) throws IOException {
CubeInstance cubeClone = cubeManager.createCube(name, projectName, cube.getDescriptor(), cube.getOwner());
CubeUpdate cubeUpdate = new CubeUpdate(cubeClone.latestCopyForWrite());
cubeUpdate.setToAddSegs(cube.getSegments().toArray(new CubeSegment[cube.getSegments().size()]));
cubeUpdate.setStatus(RealizationStatusEnum.READY);
return cubeManager.updateCube(cubeUpdate);
}
Example 13
| Source File: UpdateDictionaryStep.java From kylin-on-parquet-v2 with Apache License 2.0 | 4 votes |
|
@Override
protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
final CubeManager cubeMgr = CubeManager.getInstance(context.getConfig());
final DictionaryManager dictMgrHdfs;
final DictionaryManager dictMgrHbase;
final CubeInstance cube = cubeMgr.getCube(CubingExecutableUtil.getCubeName(this.getParams()));
final CubeSegment newSegment = cube.getSegmentById(CubingExecutableUtil.getSegmentId(this.getParams()));
final List<CubeSegment> mergingSegments = getMergingSegments(cube);
final String dictInfoPath = this.getParams().get(BatchConstants.ARG_DICT_PATH);
final String metadataUrl = this.getParams().get(BatchConstants.ARG_META_URL);
final KylinConfig kylinConfHbase = cube.getConfig();
final KylinConfig kylinConfHdfs = AbstractHadoopJob.loadKylinConfigFromHdfs(metadataUrl);
Collections.sort(mergingSegments);
try {
Configuration conf = HadoopUtil.getCurrentConfiguration();
FileSystem fs = HadoopUtil.getWorkingFileSystem();
ResourceStore hbaseRS = ResourceStore.getStore(kylinConfHbase);
ResourceStore hdfsRS = ResourceStore.getStore(kylinConfHdfs);
dictMgrHdfs = DictionaryManager.getInstance(kylinConfHdfs);
dictMgrHbase = DictionaryManager.getInstance(kylinConfHbase);
// work on copy instead of cached objects
CubeInstance cubeCopy = cube.latestCopyForWrite();
CubeSegment newSegCopy = cubeCopy.getSegmentById(newSegment.getUuid());
// update cube segment dictionary
FileStatus[] fileStatuss = fs.listStatus(new Path(dictInfoPath), new PathFilter() {
@Override
public boolean accept(Path path) {
return path.getName().startsWith("part") || path.getName().startsWith("tmp");
}
});
for (FileStatus fileStatus : fileStatuss) {
Path filePath = fileStatus.getPath();
SequenceFile.Reader reader = new SequenceFile.Reader(fs, filePath, conf);
Text key = (Text) ReflectionUtils.newInstance(reader.getKeyClass(), conf);
Text value = (Text) ReflectionUtils.newInstance(reader.getValueClass(), conf);
while (reader.next(key, value)) {
String tblCol = key.toString();
String dictInfoResource = value.toString();
if (StringUtils.isNotEmpty(dictInfoResource)) {
logger.info(dictInfoResource);
// put dictionary file to metadata store
DictionaryInfo dictInfoHdfs = dictMgrHdfs.getDictionaryInfo(dictInfoResource);
DictionaryInfo dicInfoHbase = dictMgrHbase.trySaveNewDict(dictInfoHdfs.getDictionaryObject(), dictInfoHdfs);
if (dicInfoHbase != null){
TblColRef tblColRef = cube.getDescriptor().findColumnRef(tblCol.split(":")[0], tblCol.split(":")[1]);
newSegCopy.putDictResPath(tblColRef, dicInfoHbase.getResourcePath());
}
}
}
IOUtils.closeStream(reader);
}
CubeSegment lastSeg = mergingSegments.get(mergingSegments.size() - 1);
for (Map.Entry<String, String> entry : lastSeg.getSnapshots().entrySet()) {
newSegCopy.putSnapshotResPath(entry.getKey(), entry.getValue());
}
// update statistics
// put the statistics to metadata store
String statisticsFileName = newSegment.getStatisticsResourcePath();
hbaseRS.putResource(statisticsFileName, hdfsRS.getResource(newSegment.getStatisticsResourcePath()).content(), System.currentTimeMillis());
CubeUpdate update = new CubeUpdate(cubeCopy);
update.setToUpdateSegs(newSegCopy);
cubeMgr.updateCube(update);
return ExecuteResult.createSucceed();
} catch (IOException e) {
logger.error("fail to merge dictionary", e);
return ExecuteResult.createError(e);
}
}
Example 14
| Source File: AfterMergeOrRefreshResourceMerger.java From kylin-on-parquet-v2 with Apache License 2.0 | 4 votes |
|
@Override
public void merge(String cubeId, String segmentId, ResourceStore remoteResourceStore, String jobType) {
CubeManager cubeManager = CubeManager.getInstance(getConfig());
CubeInstance cubeInstance = cubeManager.getCubeByUuid(cubeId);
CubeUpdate update = new CubeUpdate(cubeInstance.latestCopyForWrite());
CubeManager distManager = CubeManager.getInstance(remoteResourceStore.getConfig());
CubeInstance distCube = distManager.getCubeByUuid(cubeId).latestCopyForWrite();
List<CubeSegment> toUpdateSegments = Lists.newArrayList();
CubeSegment mergedSegment = distCube.getSegmentById(segmentId);
mergedSegment.setStatus(SegmentStatusEnum.READY);
Map<String, String> additionalInfo = mergedSegment.getAdditionalInfo();
additionalInfo.put("storageType", "" + IStorageAware.ID_PARQUET);
mergedSegment.setAdditionalInfo(additionalInfo);
toUpdateSegments.add(mergedSegment);
List<CubeSegment> toRemoveSegments = getToRemoveSegs(distCube, mergedSegment);
Collections.sort(toRemoveSegments);
makeSnapshotForNewSegment(mergedSegment, toRemoveSegments);
if (String.valueOf(JobTypeEnum.INDEX_MERGE).equals(jobType)) {
Optional<Long> reduce = toRemoveSegments.stream().map(CubeSegment::getSizeKB).filter(size -> size != -1)
.reduce(Long::sum);
Optional<Long> inputRecords = toRemoveSegments.stream().map(CubeSegment::getInputRecords).filter(records -> records != -1)
.reduce(Long::sum);
if (reduce.isPresent()) {
long totalSourceSize = reduce.get();
mergedSegment.setSizeKB(totalSourceSize);
mergedSegment.setInputRecords(inputRecords.get());
mergedSegment.setLastBuildTime(System.currentTimeMillis());
}
}
update.setToRemoveSegs(toRemoveSegments.toArray(new CubeSegment[0]));
update.setToUpdateSegs(toUpdateSegments.toArray(new CubeSegment[0]));
try {
cubeManager.updateCube(update);
} catch (IOException e) {
e.printStackTrace();
}
}
Example 15
| Source File: JobStepFactoryTest.java From kylin-on-parquet-v2 with Apache License 2.0 | 4 votes |
|
@Test
public void testAddStepInMerging() throws Exception {
CubeManager cubeMgr = CubeManager.getInstance(config);
CubeInstance cube = cubeMgr.getCube(CUBE_NAME);
cleanupSegments(CUBE_NAME);
/**
* Round1. Add 2 segment
*/
CubeSegment segment1 = cubeMgr.appendSegment(cube, new SegmentRange.TSRange(dateToLong("2010-01-01"), dateToLong("2013-01-01")));
CubeSegment segment2 = cubeMgr.appendSegment(cube, new SegmentRange.TSRange(dateToLong("2013-01-01"), dateToLong("2015-01-01")));
segment1.setStatus(SegmentStatusEnum.READY);
segment2.setStatus(SegmentStatusEnum.READY);
CubeInstance reloadCube = cube.latestCopyForWrite();
Segments segments = new Segments();
segments.add(segment1);
segments.add(segment2);
reloadCube.setSegments(segments);
CubeUpdate update = new CubeUpdate(reloadCube);
cubeMgr.updateCube(update);
/**
* Round2. Merge two segments
*/
reloadCube = cubeMgr.reloadCube(CUBE_NAME);
CubeSegment mergedSegment = cubeMgr.mergeSegments(reloadCube, new SegmentRange.TSRange(dateToLong("2010-01-01"), dateToLong("2015-01-01"))
, null, true);
NSparkMergingJob job = NSparkMergingJob.merge(mergedSegment, "ADMIN");
Assert.assertEquals(CUBE_NAME, job.getParam(MetadataConstants.P_CUBE_NAME));
NSparkExecutable resourceDetectStep = job.getResourceDetectStep();
Assert.assertEquals(ResourceDetectBeforeMergingJob.class.getName(),
resourceDetectStep.getSparkSubmitClassName());
Assert.assertEquals(ExecutableConstants.STEP_NAME_DETECT_RESOURCE, resourceDetectStep.getName());
job.getParams().forEach((key, value) -> Assert.assertEquals(value, resourceDetectStep.getParam(key)));
Assert.assertEquals(config.getJobTmpMetaStoreUrl(getProject(), resourceDetectStep.getId()).toString(),
resourceDetectStep.getDistMetaUrl());
NSparkExecutable mergeStep = job.getSparkMergingStep();
Assert.assertEquals(config.getSparkMergeClassName(), mergeStep.getSparkSubmitClassName());
Assert.assertEquals(ExecutableConstants.STEP_NAME_MERGER_SPARK_SEGMENT, mergeStep.getName());
job.getParams().forEach((key, value) -> Assert.assertEquals(value, mergeStep.getParam(key)));
Assert.assertEquals(config.getJobTmpMetaStoreUrl(getProject(), mergeStep.getId()).toString(),
mergeStep.getDistMetaUrl());
CubeInstance cubeInstance = cubeMgr.reloadCube(CUBE_NAME);
NSparkUpdateMetaAndCleanupAfterMergeStep cleanStep = job.getCleanUpAfterMergeStep();
job.getParams().forEach((key, value) -> {
if (key.equalsIgnoreCase(MetadataConstants.P_SEGMENT_IDS)) {
final List<String> needDeleteSegmentNames = cubeInstance.getMergingSegments(mergedSegment).stream()
.map(CubeSegment::getName).collect(Collectors.toList());
Assert.assertEquals(needDeleteSegmentNames, Arrays.asList(cleanStep.getParam(MetadataConstants.P_SEGMENT_NAMES).split(",")));
} else {
Assert.assertEquals(value, mergeStep.getParam(key));
}
});
Assert.assertEquals(config.getJobTmpMetaStoreUrl(getProject(), cleanStep.getId()).toString(),
cleanStep.getDistMetaUrl());
}
Example 16
| Source File: UpdateDictionaryStep.java From kylin with Apache License 2.0 | 4 votes |
|
@Override
protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
final CubeManager cubeMgr = CubeManager.getInstance(context.getConfig());
final DictionaryManager dictMgrHdfs;
final DictionaryManager dictMgrHbase;
final CubeInstance cube = cubeMgr.getCube(CubingExecutableUtil.getCubeName(this.getParams()));
final CubeSegment newSegment = cube.getSegmentById(CubingExecutableUtil.getSegmentId(this.getParams()));
final List<CubeSegment> mergingSegments = getMergingSegments(cube);
final String dictInfoPath = this.getParams().get(BatchConstants.ARG_DICT_PATH);
final String metadataUrl = this.getParams().get(BatchConstants.ARG_META_URL);
final KylinConfig kylinConfHbase = cube.getConfig();
final KylinConfig kylinConfHdfs = AbstractHadoopJob.loadKylinConfigFromHdfs(metadataUrl);
Collections.sort(mergingSegments);
try {
Configuration conf = HadoopUtil.getCurrentConfiguration();
FileSystem fs = HadoopUtil.getWorkingFileSystem();
ResourceStore hbaseRS = ResourceStore.getStore(kylinConfHbase);
ResourceStore hdfsRS = ResourceStore.getStore(kylinConfHdfs);
dictMgrHdfs = DictionaryManager.getInstance(kylinConfHdfs);
dictMgrHbase = DictionaryManager.getInstance(kylinConfHbase);
// work on copy instead of cached objects
CubeInstance cubeCopy = cube.latestCopyForWrite();
CubeSegment newSegCopy = cubeCopy.getSegmentById(newSegment.getUuid());
// update cube segment dictionary
FileStatus[] fileStatuss = fs.listStatus(new Path(dictInfoPath), new PathFilter() {
@Override
public boolean accept(Path path) {
return path.getName().startsWith("part") || path.getName().startsWith("tmp");
}
});
for (FileStatus fileStatus : fileStatuss) {
Path filePath = fileStatus.getPath();
SequenceFile.Reader reader = new SequenceFile.Reader(fs, filePath, conf);
Text key = (Text) ReflectionUtils.newInstance(reader.getKeyClass(), conf);
Text value = (Text) ReflectionUtils.newInstance(reader.getValueClass(), conf);
while (reader.next(key, value)) {
String tblCol = key.toString();
String dictInfoResource = value.toString();
if (StringUtils.isNotEmpty(dictInfoResource)) {
logger.info(dictInfoResource);
// put dictionary file to metadata store
DictionaryInfo dictInfoHdfs = dictMgrHdfs.getDictionaryInfo(dictInfoResource);
DictionaryInfo dicInfoHbase = dictMgrHbase.trySaveNewDict(dictInfoHdfs.getDictionaryObject(), dictInfoHdfs);
if (dicInfoHbase != null){
TblColRef tblColRef = cube.getDescriptor().findColumnRef(tblCol.split(":")[0], tblCol.split(":")[1]);
newSegCopy.putDictResPath(tblColRef, dicInfoHbase.getResourcePath());
}
}
}
IOUtils.closeStream(reader);
}
CubeSegment lastSeg = mergingSegments.get(mergingSegments.size() - 1);
for (Map.Entry<String, String> entry : lastSeg.getSnapshots().entrySet()) {
newSegCopy.putSnapshotResPath(entry.getKey(), entry.getValue());
}
// update statistics
// put the statistics to metadata store
String statisticsFileName = newSegment.getStatisticsResourcePath();
hbaseRS.putResource(statisticsFileName, hdfsRS.getResource(newSegment.getStatisticsResourcePath()).content(), System.currentTimeMillis());
CubeUpdate update = new CubeUpdate(cubeCopy);
update.setToUpdateSegs(newSegCopy);
cubeMgr.updateCube(update);
return ExecuteResult.createSucceed();
} catch (IOException e) {
logger.error("fail to merge dictionary", e);
return ExecuteResult.createError(e);
}
}
