Java Code Examples for org.apache.kylin.metadata.model.PartitionDesc#getPartitionDateColumnRef()

The following examples show how to use org.apache.kylin.metadata.model.PartitionDesc#getPartitionDateColumnRef() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: KeyValueBuilder.java    From kylin with Apache License 2.0 6 votes vote down vote up 
/**
 * Use the segment start time as the map key, the time unit depends on the partition columns
 * If the partition_time_column is null, the unit is day;
 *                            otherwise, the unit is second
 */
private String getSegmentStartTime(CubeSegment segment) {
    long startTime = segment.getTSRange().start.v;
    DataModelDesc model = segment.getModel();
    PartitionDesc partitionDesc = model.getPartitionDesc();
    if (partitionDesc == null || !partitionDesc.isPartitioned()) {
        return "0";
    } else if (partitionDesc.partitionColumnIsTimeMillis()) {
        return "" + startTime;
    } else if (partitionDesc.getPartitionTimeColumnRef() != null) {
        return "" + startTime / 1000L;
    } else if (partitionDesc.getPartitionDateColumnRef() != null) {
        return "" + startTime / 86400000L;
    }
    return "0";
}
Example 2
Source File: HBaseStorage.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
private static TblColRef getPartitionCol(IRealization realization) {
    String modelName = realization.getModel().getName();
    DataModelDesc dataModelDesc = DataModelManager.getInstance(KylinConfig.getInstanceFromEnv())
            .getDataModelDesc(modelName);
    PartitionDesc partitionDesc = dataModelDesc.getPartitionDesc();
    Preconditions.checkArgument(partitionDesc != null, "PartitionDesc for " + realization + " is null!");
    TblColRef partitionColRef = partitionDesc.getPartitionDateColumnRef();
    Preconditions.checkArgument(partitionColRef != null,
            "getPartitionDateColumnRef for " + realization + " is null");
    return partitionColRef;
}
Example 3
Source File: HBaseStorage.java    From kylin with Apache License 2.0 5 votes vote down vote up 
private static TblColRef getPartitionCol(IRealization realization) {
    String modelName = realization.getModel().getName();
    DataModelDesc dataModelDesc = DataModelManager.getInstance(KylinConfig.getInstanceFromEnv())
            .getDataModelDesc(modelName);
    PartitionDesc partitionDesc = dataModelDesc.getPartitionDesc();
    Preconditions.checkArgument(partitionDesc != null, "PartitionDesc for " + realization + " is null!");
    TblColRef partitionColRef = partitionDesc.getPartitionDateColumnRef();
    Preconditions.checkArgument(partitionColRef != null,
            "getPartitionDateColumnRef for " + realization + " is null");
    return partitionColRef;
}
Example 4
Source File: GTCubeStorageQueryBase.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up 
private boolean isExactAggregation(StorageContext context, Cuboid cuboid, Collection<TblColRef> groups,
        Set<TblColRef> othersD, Set<TblColRef> singleValuesD, Set<TblColRef> derivedPostAggregation,
        Collection<FunctionDesc> functionDescs, List<SQLDigest.SQLCall> aggrSQLCalls, boolean groupByExpression) {
    if (context.isNeedStorageAggregation()) {
        logger.info("exactAggregation is false because need storage aggregation");
        return false;
    }

    if (cuboid.requirePostAggregation()) {
        logger.info("exactAggregation is false because cuboid {}=>{}", cuboid.getInputID(), cuboid.getId());
        return false;
    }

    // derived aggregation is bad, unless expanded columns are already in group by
    if (!groups.containsAll(derivedPostAggregation)) {
        logger.info("exactAggregation is false because derived column require post aggregation: {}",
                derivedPostAggregation);
        return false;
    }

    // other columns (from filter) is bad, unless they are ensured to have single value
    if (!singleValuesD.containsAll(othersD)) {
        logger.info("exactAggregation is false because some column not on group by: {} (single value column: {})",
                othersD, singleValuesD);
        return false;
    }

    //for DimensionAsMetric like max(cal_dt), the dimension column maybe not in real group by
    for (FunctionDesc functionDesc : functionDescs) {
        if (functionDesc.isDimensionAsMetric()) {
            logger.info("exactAggregation is false because has DimensionAsMetric");
            return false;
        }
    }
    for (SQLDigest.SQLCall aggrSQLCall : aggrSQLCalls) {
        if (aggrSQLCall.function.equals(BitmapMeasureType.FUNC_INTERSECT_COUNT_DISTINCT)
        || aggrSQLCall.function.equals(BitmapMeasureType.FUNC_INTERSECT_VALUE)) {
            logger.info("exactAggregation is false because has INTERSECT_COUNT OR INTERSECT_VALUE");
            return false;
        }
    }

    // for partitioned cube, the partition column must belong to group by or has single value
    PartitionDesc partDesc = cuboid.getCubeDesc().getModel().getPartitionDesc();
    if (partDesc.isPartitioned()) {
        TblColRef col = partDesc.getPartitionDateColumnRef();
        if (!groups.contains(col) && !singleValuesD.contains(col)) {
            logger.info("exactAggregation is false because cube is partitioned and {} is not on group by", col);
            return false;
        }
    }

    // for group by expression like: group by seller_id/100. seller_id_1(200) get 2, seller_id_2(201) also get 2, so can't aggregate exactly
    if (groupByExpression) {
        logger.info("exactAggregation is false because group by expression");
        return false;
    }

    logger.info("exactAggregation is true, cuboid id is {}", cuboid.getId());
    return true;
}
Example 5
Source File: GTCubeStorageQueryBase.java    From kylin with Apache License 2.0 4 votes vote down vote up 
private boolean isExactAggregation(StorageContext context, Cuboid cuboid, Collection<TblColRef> groups,
        Set<TblColRef> othersD, Set<TblColRef> singleValuesD, Set<TblColRef> derivedPostAggregation,
        Collection<FunctionDesc> functionDescs, List<SQLDigest.SQLCall> aggrSQLCalls, boolean groupByExpression) {
    if (context.isNeedStorageAggregation()) {
        logger.info("exactAggregation is false because need storage aggregation");
        return false;
    }

    if (cuboid.requirePostAggregation()) {
        logger.info("exactAggregation is false because cuboid {}=>{}", cuboid.getInputID(), cuboid.getId());
        return false;
    }

    // derived aggregation is bad, unless expanded columns are already in group by
    if (!groups.containsAll(derivedPostAggregation)) {
        logger.info("exactAggregation is false because derived column require post aggregation: {}",
                derivedPostAggregation);
        return false;
    }

    // other columns (from filter) is bad, unless they are ensured to have single value
    if (!singleValuesD.containsAll(othersD)) {
        logger.info("exactAggregation is false because some column not on group by: {} (single value column: {})",
                othersD, singleValuesD);
        return false;
    }

    //for DimensionAsMetric like max(cal_dt), the dimension column maybe not in real group by
    for (FunctionDesc functionDesc : functionDescs) {
        if (functionDesc.isDimensionAsMetric()) {
            logger.info("exactAggregation is false because has DimensionAsMetric");
            return false;
        }
    }
    for (SQLDigest.SQLCall aggrSQLCall : aggrSQLCalls) {
        if (aggrSQLCall.function.equals(BitmapMeasureType.FUNC_INTERSECT_COUNT_DISTINCT)
        || aggrSQLCall.function.equals(BitmapMeasureType.FUNC_INTERSECT_VALUE)) {
            logger.info("exactAggregation is false because has INTERSECT_COUNT OR INTERSECT_VALUE");
            return false;
        }
    }

    // for partitioned cube, the partition column must belong to group by or has single value
    PartitionDesc partDesc = cuboid.getCubeDesc().getModel().getPartitionDesc();
    if (partDesc.isPartitioned()) {
        TblColRef col = partDesc.getPartitionDateColumnRef();
        if (!groups.contains(col) && !singleValuesD.contains(col)) {
            logger.info("exactAggregation is false because cube is partitioned and {} is not on group by", col);
            return false;
        }
    }

    // for group by expression like: group by seller_id/100. seller_id_1(200) get 2, seller_id_2(201) also get 2, so can't aggregate exactly
    if (groupByExpression) {
        logger.info("exactAggregation is false because group by expression");
        return false;
    }

    logger.info("exactAggregation is true, cuboid id is {}", cuboid.getId());
    return true;
}
Example 6
Source File: TableSchemaUpdater.java    From kylin with Apache License 2.0 4 votes vote down vote up 
public static DataModelDesc dealWithMappingForModel(DataModelDesc other,
        Map<String, TableSchemaUpdateMapping> mappings) {
    // For filter condition, not support
    if (!Strings.isNullOrEmpty(other.getFilterCondition())) {
        throw new UnsupportedOperationException("Cannot deal with filter condition " + other.getFilterCondition());
    }

    DataModelDesc copy = DataModelDesc.getCopyOf(other);
    copy.setLastModified(other.getLastModified());

    // mapping for root fact table identity
    TableSchemaUpdateMapping rootMapping = getTableSchemaUpdateMapping(mappings, other.getRootFactTableName());
    if (rootMapping != null) {
        TableDesc rootFactTable = other.getRootFactTable().getTableDesc();
        copy.setRootFactTableName(
                rootMapping.getTableIdentity(rootFactTable.getDatabase(), rootFactTable.getName()));
    }

    // mapping for joins
    JoinTableDesc[] joinTables = other.getJoinTables();
    JoinTableDesc[] joinTablesCopy = new JoinTableDesc[joinTables.length];
    for (int i = 0; i < joinTables.length; i++) {
        JoinTableDesc joinTable = joinTables[i];
        joinTablesCopy[i] = JoinTableDesc.getCopyOf(joinTable);
        String tableIdentity = joinTable.getTable();
        TableSchemaUpdateMapping mapping = getTableSchemaUpdateMapping(mappings, tableIdentity);
        if (mapping != null && mapping.isTableIdentityChanged()) {
            joinTablesCopy[i].setTable(mapping.getTableIdentity(tableIdentity));
        }
    }
    copy.setJoinTables(joinTablesCopy);

    // mapping for partition columns
    PartitionDesc partDesc = other.getPartitionDesc();
    PartitionDesc partCopy = PartitionDesc.getCopyOf(partDesc);
    if (partDesc.getPartitionDateColumnRef() != null) {
        partCopy.setPartitionDateColumn(
                replacePartitionCol(partDesc.getPartitionDateColumnRef().getCanonicalName(), mappings));
    }
    if (partDesc.getPartitionTimeColumnRef() != null) {
        partCopy.setPartitionTimeColumn(
                replacePartitionCol(partDesc.getPartitionTimeColumnRef().getCanonicalName(), mappings));
    }
    copy.setPartitionDesc(partCopy);

    return copy;
}