Java Code Examples for org.apache.kylin.common.persistence.ResourceStore#exists()

The following examples show how to use org.apache.kylin.common.persistence.ResourceStore#exists() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: SparkExecutable.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up 
private void attachSegmentsMetadataWithDict(List<CubeSegment> segments) throws IOException {
    Set<String> dumpList = new LinkedHashSet<>(
            JobRelatedMetaUtil.collectCubeMetadata(segments.get(0).getCubeInstance()));
    ResourceStore rs = ResourceStore.getStore(segments.get(0).getConfig());
    for (CubeSegment segment : segments) {
        dumpList.addAll(segment.getDictionaryPaths());
        if (rs.exists(segment.getStatisticsResourcePath())) {
            // cube statistics is not available for new segment
            dumpList.add(segment.getStatisticsResourcePath());
        }
        //tiretree global domain dic
        CubeDescTiretreeGlobalDomainDictUtil.cuboidJob(segment.getCubeDesc(), dumpList);
    }
    JobRelatedMetaUtil.dumpAndUploadKylinPropsAndMetadata(dumpList, (KylinConfigExt) segments.get(0).getConfig(),
            this.getParam(SparkCubingByLayer.OPTION_META_URL.getOpt()));
}
Example 2
Source File: SparkExecutable.java    From kylin with Apache License 2.0 6 votes vote down vote up 
private void attachSegmentsMetadataWithDict(List<CubeSegment> segments) throws IOException {
    Set<String> dumpList = new LinkedHashSet<>(
            JobRelatedMetaUtil.collectCubeMetadata(segments.get(0).getCubeInstance()));
    ResourceStore rs = ResourceStore.getStore(segments.get(0).getConfig());
    for (CubeSegment segment : segments) {
        dumpList.addAll(segment.getDictionaryPaths());
        if (rs.exists(segment.getStatisticsResourcePath())) {
            // cube statistics is not available for new segment
            dumpList.add(segment.getStatisticsResourcePath());
        }
        //tiretree global domain dic
        CubeDescTiretreeGlobalDomainDictUtil.cuboidJob(segment.getCubeDesc(), dumpList);
    }
    JobRelatedMetaUtil.dumpAndUploadKylinPropsAndMetadata(dumpList, (KylinConfigExt) segments.get(0).getConfig(),
            this.getParam(SparkCubingByLayer.OPTION_META_URL.getOpt()));
}
Example 3
Source File: FlinkExecutable.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
private void attachSegmentMetadataWithDict(CubeSegment segment) throws IOException {
    Set<String> dumpList = new LinkedHashSet<>();
    dumpList.addAll(JobRelatedMetaUtil.collectCubeMetadata(segment.getCubeInstance()));
    dumpList.addAll(segment.getDictionaryPaths());
    ResourceStore rs = ResourceStore.getStore(segment.getConfig());
    if (rs.exists(segment.getStatisticsResourcePath())) {
        // cube statistics is not available for new segment
        dumpList.add(segment.getStatisticsResourcePath());
    }

    JobRelatedMetaUtil.dumpAndUploadKylinPropsAndMetadata(dumpList, (KylinConfigExt) segment.getConfig(),
            this.getParam(FlinkCubingByLayer.OPTION_META_URL.getOpt()));
}
Example 4
Source File: FlinkExecutable.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
private void attachSegmentsMetadataWithDict(List<CubeSegment> segments) throws IOException {
    Set<String> dumpList = new LinkedHashSet<>();
    dumpList.addAll(JobRelatedMetaUtil.collectCubeMetadata(segments.get(0).getCubeInstance()));
    ResourceStore rs = ResourceStore.getStore(segments.get(0).getConfig());
    for (CubeSegment segment : segments) {
        dumpList.addAll(segment.getDictionaryPaths());
        if (rs.exists(segment.getStatisticsResourcePath())) {
            // cube statistics is not available for new segment
            dumpList.add(segment.getStatisticsResourcePath());
        }
    }

    JobRelatedMetaUtil.dumpAndUploadKylinPropsAndMetadata(dumpList, (KylinConfigExt) segments.get(0).getConfig(),
            this.getParam(FlinkCubingByLayer.OPTION_META_URL.getOpt()));
}
Example 5
Source File: SparkExecutable.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
private void attachSegmentMetadataWithDict(CubeSegment segment) throws IOException {
    Set<String> dumpList = new LinkedHashSet<>();
    dumpList.addAll(JobRelatedMetaUtil.collectCubeMetadata(segment.getCubeInstance()));
    dumpList.addAll(segment.getDictionaryPaths());
    ResourceStore rs = ResourceStore.getStore(segment.getConfig());
    if (rs.exists(segment.getStatisticsResourcePath())) {
        // cube statistics is not available for new segment
        dumpList.add(segment.getStatisticsResourcePath());
    }
    //tiretree global domain dic
    CubeDescTiretreeGlobalDomainDictUtil.cuboidJob(segment.getCubeDesc(), dumpList);

    JobRelatedMetaUtil.dumpAndUploadKylinPropsAndMetadata(dumpList, (KylinConfigExt) segment.getConfig(),
            this.getParam(SparkCubingByLayer.OPTION_META_URL.getOpt()));
}
Example 6
Source File: FlinkExecutable.java    From kylin with Apache License 2.0 5 votes vote down vote up 
private void attachSegmentMetadataWithDict(CubeSegment segment) throws IOException {
    Set<String> dumpList = new LinkedHashSet<>();
    dumpList.addAll(JobRelatedMetaUtil.collectCubeMetadata(segment.getCubeInstance()));
    dumpList.addAll(segment.getDictionaryPaths());
    ResourceStore rs = ResourceStore.getStore(segment.getConfig());
    if (rs.exists(segment.getStatisticsResourcePath())) {
        // cube statistics is not available for new segment
        dumpList.add(segment.getStatisticsResourcePath());
    }

    JobRelatedMetaUtil.dumpAndUploadKylinPropsAndMetadata(dumpList, (KylinConfigExt) segment.getConfig(),
            this.getParam(FlinkCubingByLayer.OPTION_META_URL.getOpt()));
}
Example 7
Source File: FlinkExecutable.java    From kylin with Apache License 2.0 5 votes vote down vote up 
private void attachSegmentsMetadataWithDict(List<CubeSegment> segments) throws IOException {
    Set<String> dumpList = new LinkedHashSet<>();
    dumpList.addAll(JobRelatedMetaUtil.collectCubeMetadata(segments.get(0).getCubeInstance()));
    ResourceStore rs = ResourceStore.getStore(segments.get(0).getConfig());
    for (CubeSegment segment : segments) {
        dumpList.addAll(segment.getDictionaryPaths());
        if (rs.exists(segment.getStatisticsResourcePath())) {
            // cube statistics is not available for new segment
            dumpList.add(segment.getStatisticsResourcePath());
        }
    }

    JobRelatedMetaUtil.dumpAndUploadKylinPropsAndMetadata(dumpList, (KylinConfigExt) segments.get(0).getConfig(),
            this.getParam(FlinkCubingByLayer.OPTION_META_URL.getOpt()));
}
Example 8
Source File: MigrationRuleSet.java    From kylin with Apache License 2.0 5 votes vote down vote up 
@Override
public void apply(Context ctx) throws RuleValidationException {
    ResourceStore dstStore = ctx.getTargetResourceStore();
    CubeInstance cube = ctx.getCubeInstance();
    try {
        if (dstStore.exists(cube.getResourcePath()))
            throw new RuleValidationException("The cube named " + cube.getName()
                    + " already exists on target metadata store. Please delete it firstly and try again");
    } catch (IOException e) {
        logger.error(e.getMessage(), e);
        throw new RuleValidationException(e.getMessage(), e);
    }
}
Example 9
Source File: SparkExecutable.java    From kylin with Apache License 2.0 5 votes vote down vote up 
private void attachSegmentMetadataWithDict(CubeSegment segment) throws IOException {
    Set<String> dumpList = new LinkedHashSet<>();
    dumpList.addAll(JobRelatedMetaUtil.collectCubeMetadata(segment.getCubeInstance()));
    dumpList.addAll(segment.getDictionaryPaths());
    ResourceStore rs = ResourceStore.getStore(segment.getConfig());
    if (rs.exists(segment.getStatisticsResourcePath())) {
        // cube statistics is not available for new segment
        dumpList.add(segment.getStatisticsResourcePath());
    }
    //tiretree global domain dic
    CubeDescTiretreeGlobalDomainDictUtil.cuboidJob(segment.getCubeDesc(), dumpList);

    JobRelatedMetaUtil.dumpAndUploadKylinPropsAndMetadata(dumpList, (KylinConfigExt) segment.getConfig(),
            this.getParam(SparkCubingByLayer.OPTION_META_URL.getOpt()));
}
Example 10
Source File: AclTableMigrationTool.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up 
private boolean isTableAlreadyMigrate(ResourceStore store, String tableName) throws IOException {
    return store.exists(MIGRATE_OK_PREFIX + tableName);
}
Example 11
Source File: AclTableMigrationTool.java    From kylin with Apache License 2.0 4 votes vote down vote up 
private boolean isTableAlreadyMigrate(ResourceStore store, String tableName) throws IOException {
    return store.exists(MIGRATE_OK_PREFIX + tableName);
}