Java Code Examples for org.apache.kylin.metadata.model.DataModelDesc#getPartitionDesc()

The following examples show how to use org.apache.kylin.metadata.model.DataModelDesc#getPartitionDesc() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: JoinedFormatter.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up 
private void setDateEnv(IJoinedFlatTableDesc flatDesc) {
    DataModelDesc model = flatDesc.getDataModel();
    PartitionDesc partDesc = model.getPartitionDesc();
    SegmentRange segRange = flatDesc.getSegRange();
    long startInclusive = (Long) segRange.start.v;
    long endExclusive = (Long) segRange.end.v;
    //
    String startDate = "";
    String endDate = "";
    String partitionColumnDateFormat = partDesc.getPartitionDateFormat();
    if (partDesc.getPartitionTimeColumn() == null && partDesc.getPartitionDateColumn() == null) {
        startDate = String.valueOf(startInclusive);
        endDate = String.valueOf(endExclusive);
    } else {
        startDate = DateFormat.formatToDateStr(startInclusive, partitionColumnDateFormat);
        endDate = DateFormat.formatToDateStr(endExclusive, partitionColumnDateFormat);
    }
    setKeyValue(START_DATE, startDate);
    setKeyValue(END_DATE, endDate);
}
Example 2
Source File: DataModelDescResponse.java    From kylin with Apache License 2.0 6 votes vote down vote up 
public DataModelDescResponse(DataModelDesc dataModelDesc) {
    setUuid(dataModelDesc.getUuid());
    setLastModified(dataModelDesc.getLastModified());
    setVersion(dataModelDesc.getVersion());
    setName(dataModelDesc.getName());
    setOwner(dataModelDesc.getOwner());
    setDraft(dataModelDesc.isDraft());
    setDescription(dataModelDesc.getDescription());
    setRootFactTableName(dataModelDesc.getRootFactTableName());
    setJoinTables(dataModelDesc.getJoinTables());
    setDimensions(dataModelDesc.getDimensions());
    setMetrics(dataModelDesc.getMetrics());
    setFilterCondition(dataModelDesc.getFilterCondition());
    if (dataModelDesc.getPartitionDesc() != null)
        setPartitionDesc(PartitionDesc.getCopyOf(dataModelDesc.getPartitionDesc()));
    setCapacity(dataModelDesc.getCapacity());
}
Example 3
Source File: SegmentPruner.java    From kylin with Apache License 2.0 6 votes vote down vote up 
public static DimensionRangeInfo tryDeduceRangeFromPartitionCol(CubeSegment seg, TblColRef col) {
    DataModelDesc model = seg.getModel();
    PartitionDesc part = model.getPartitionDesc();

    if (!part.isPartitioned())
        return null;
    if (!col.equals(part.getPartitionDateColumnRef()))
        return null;

    // deduce the dim range from TSRange
    TSRange tsRange = seg.getTSRange();
    if (tsRange.start.isMin || tsRange.end.isMax)
        return null; // DimensionRangeInfo cannot express infinite

    String min = tsRangeToStr(tsRange.start.v, part);
    String max = tsRangeToStr(tsRange.end.v - 1, part); // note the -1, end side is exclusive
    return new DimensionRangeInfo(min, max);
}
Example 4
Source File: KeyValueBuilder.java    From kylin with Apache License 2.0 6 votes vote down vote up 
/**
 * Use the segment start time as the map key, the time unit depends on the partition columns
 * If the partition_time_column is null, the unit is day;
 *                            otherwise, the unit is second
 */
private String getSegmentStartTime(CubeSegment segment) {
    long startTime = segment.getTSRange().start.v;
    DataModelDesc model = segment.getModel();
    PartitionDesc partitionDesc = model.getPartitionDesc();
    if (partitionDesc == null || !partitionDesc.isPartitioned()) {
        return "0";
    } else if (partitionDesc.partitionColumnIsTimeMillis()) {
        return "" + startTime;
    } else if (partitionDesc.getPartitionTimeColumnRef() != null) {
        return "" + startTime / 1000L;
    } else if (partitionDesc.getPartitionDateColumnRef() != null) {
        return "" + startTime / 86400000L;
    }
    return "0";
}
Example 5
Source File: JoinedFormatter.java    From kylin with Apache License 2.0 6 votes vote down vote up 
private void setDateEnv(IJoinedFlatTableDesc flatDesc) {
    DataModelDesc model = flatDesc.getDataModel();
    PartitionDesc partDesc = model.getPartitionDesc();
    SegmentRange segRange = flatDesc.getSegRange();
    long startInclusive = (Long) segRange.start.v;
    long endExclusive = (Long) segRange.end.v;
    //
    String startDate = "";
    String endDate = "";
    String partitionColumnDateFormat = partDesc.getPartitionDateFormat();
    if (partDesc.getPartitionTimeColumn() == null && partDesc.getPartitionDateColumn() == null) {
        startDate = String.valueOf(startInclusive);
        endDate = String.valueOf(endExclusive);
    } else {
        startDate = DateFormat.formatToDateStr(startInclusive, partitionColumnDateFormat);
        endDate = DateFormat.formatToDateStr(endExclusive, partitionColumnDateFormat);
    }
    setKeyValue(START_DATE, startDate);
    setKeyValue(END_DATE, endDate);
}
Example 6
Source File: DataModelDescResponse.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up 
public DataModelDescResponse(DataModelDesc dataModelDesc) {
    setUuid(dataModelDesc.getUuid());
    setLastModified(dataModelDesc.getLastModified());
    setVersion(dataModelDesc.getVersion());
    setName(dataModelDesc.getName());
    setOwner(dataModelDesc.getOwner());
    setDraft(dataModelDesc.isDraft());
    setDescription(dataModelDesc.getDescription());
    setRootFactTableName(dataModelDesc.getRootFactTableName());
    setJoinTables(dataModelDesc.getJoinTables());
    setDimensions(dataModelDesc.getDimensions());
    setMetrics(dataModelDesc.getMetrics());
    setFilterCondition(dataModelDesc.getFilterCondition());
    if (dataModelDesc.getPartitionDesc() != null)
        setPartitionDesc(PartitionDesc.getCopyOf(dataModelDesc.getPartitionDesc()));
    setCapacity(dataModelDesc.getCapacity());
}
Example 7
Source File: SegmentPruner.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up 
public static DimensionRangeInfo tryDeduceRangeFromPartitionCol(CubeSegment seg, TblColRef col) {
    DataModelDesc model = seg.getModel();
    PartitionDesc part = model.getPartitionDesc();

    if (!part.isPartitioned())
        return null;
    if (!col.equals(part.getPartitionDateColumnRef()))
        return null;

    // deduce the dim range from TSRange
    TSRange tsRange = seg.getTSRange();
    if (tsRange.start.isMin || tsRange.end.isMax)
        return null; // DimensionRangeInfo cannot express infinite

    String min = tsRangeToStr(tsRange.start.v, part);
    String max = tsRangeToStr(tsRange.end.v - 1, part); // note the -1, end side is exclusive
    return new DimensionRangeInfo(min, max);
}
Example 8
Source File: ModelService.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
public void validateModel(String project, DataModelDesc desc) throws IllegalArgumentException {
    String factTableName = desc.getRootFactTableName();
    TableDesc tableDesc = getTableManager().getTableDesc(factTableName, project);
    if ((tableDesc.getSourceType() == ISourceAware.ID_STREAMING || tableDesc.isStreamingTable())
            && (desc.getPartitionDesc() == null || desc.getPartitionDesc().getPartitionDateColumn() == null)) {
        throw new IllegalArgumentException("Must define a partition column.");
    }
}
Example 9
Source File: JdbcHiveInputBase.java    From kylin with Apache License 2.0 5 votes vote down vote up 
private static void appendWhereStatement(IJoinedFlatTableDesc flatDesc, StringBuilder sql, boolean singleLine,
        IJdbcMetadata metadata, Map<String, String> metaMap) {
    final String sep = singleLine ? " " : "\n";

    StringBuilder whereBuilder = new StringBuilder();
    whereBuilder.append("WHERE 1=1");

    DataModelDesc model = flatDesc.getDataModel();
    if (StringUtils.isNotEmpty(model.getFilterCondition())) {
        whereBuilder.append(" AND (").append(model.getFilterCondition()).append(") ");
    }

    if (flatDesc.getSegment() != null) {
        PartitionDesc partDesc = model.getPartitionDesc();
        if (partDesc != null && partDesc.getPartitionDateColumn() != null) {
            SegmentRange segRange = flatDesc.getSegRange();

            if (segRange != null && !segRange.isInfinite()) {
                whereBuilder.append(" AND (");
                whereBuilder.append(partDesc.getPartitionConditionBuilder().buildDateRangeCondition(partDesc,
                        flatDesc.getSegment(), segRange,
                        col -> getTableColumnIdentityQuoted(col, metadata, metaMap, true)));
                whereBuilder.append(")");
                whereBuilder.append(sep);
            }
        }
    }
    sql.append(whereBuilder.toString());
}
Example 10
Source File: StreamingCubeRule.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
@Override
public void validate(CubeDesc cube, ValidateContext context) {
    DataModelDesc model = cube.getModel();
    
    if (model.getRootFactTable().getTableDesc().getSourceType() != ISourceAware.ID_STREAMING
            && !model.getRootFactTable().getTableDesc().isStreamingTable()) {
        return;
    }

    if (model.getPartitionDesc() == null || model.getPartitionDesc().getPartitionDateColumn() == null) {
        context.addResult(ResultLevel.ERROR, "Must define a partition column.");
        return;
    }

    final TblColRef partitionCol = model.getPartitionDesc().getPartitionDateColumnRef();
    boolean found = false;
    for (DimensionDesc dimensionDesc : cube.getDimensions()) {
        for (TblColRef dimCol : dimensionDesc.getColumnRefs()) {
            if (dimCol.equals(partitionCol)) {
                found = true;
                break;
            }
        }
    }

    if (found == false) {
        context.addResult(ResultLevel.ERROR, "Partition column '" + partitionCol + "' isn't in dimension list.");
        return;
    }

}
Example 11
Source File: JoinedFlatTable.java    From kylin with Apache License 2.0 5 votes vote down vote up 
private static void appendWhereStatement(IJoinedFlatTableDesc flatDesc, StringBuilder sql, boolean singleLine, SqlDialect sqlDialect) {
    final String sep = singleLine ? " " : "\n";

    StringBuilder whereBuilder = new StringBuilder();
    whereBuilder.append("WHERE 1=1");

    DataModelDesc model = flatDesc.getDataModel();
    if (StringUtils.isNotEmpty(model.getFilterCondition())) {
        String filterCondition = model.getFilterCondition();
        if (flatDesc.getSegment() != null) {
            JoinedFormatter formatter = new JoinedFormatter(flatDesc);
            filterCondition = formatter.formatSentence(model.getFilterCondition());
        }
        String quotedFilterCondition = quoteIdentifierInSqlExpr(flatDesc, filterCondition, null);
        whereBuilder.append(" AND (").append(quotedFilterCondition).append(") "); // -> filter condition contains special character may cause bug
    }
    if (flatDesc.getSegment() != null) {
        PartitionDesc partDesc = model.getPartitionDesc();
        if (partDesc != null && partDesc.getPartitionDateColumn() != null) {
            SegmentRange segRange = flatDesc.getSegRange();

            if (segRange != null && !segRange.isInfinite()) {
                whereBuilder.append(" AND (");
                String quotedPartitionCond = quoteIdentifierInSqlExpr(flatDesc,
                        partDesc.getPartitionConditionBuilder().buildDateRangeCondition(partDesc, flatDesc.getSegment(), segRange, null), sqlDialect);
                whereBuilder.append(quotedPartitionCond);
                whereBuilder.append(")" + sep);
            }
        }
    }

    sql.append(whereBuilder.toString());
}
Example 12
Source File: HBaseStorage.java    From kylin with Apache License 2.0 5 votes vote down vote up 
private static TblColRef getPartitionCol(IRealization realization) {
    String modelName = realization.getModel().getName();
    DataModelDesc dataModelDesc = DataModelManager.getInstance(KylinConfig.getInstanceFromEnv())
            .getDataModelDesc(modelName);
    PartitionDesc partitionDesc = dataModelDesc.getPartitionDesc();
    Preconditions.checkArgument(partitionDesc != null, "PartitionDesc for " + realization + " is null!");
    TblColRef partitionColRef = partitionDesc.getPartitionDateColumnRef();
    Preconditions.checkArgument(partitionColRef != null,
            "getPartitionDateColumnRef for " + realization + " is null");
    return partitionColRef;
}
Example 13
Source File: StreamingCubeRule.java    From kylin with Apache License 2.0 5 votes vote down vote up 
@Override
public void validate(CubeDesc cube, ValidateContext context) {
    DataModelDesc model = cube.getModel();
    
    if (model.getRootFactTable().getTableDesc().getSourceType() != ISourceAware.ID_STREAMING
            && !model.getRootFactTable().getTableDesc().isStreamingTable()) {
        return;
    }

    if (model.getPartitionDesc() == null || model.getPartitionDesc().getPartitionDateColumn() == null) {
        context.addResult(ResultLevel.ERROR, "Must define a partition column.");
        return;
    }

    final TblColRef partitionCol = model.getPartitionDesc().getPartitionDateColumnRef();
    boolean found = false;
    for (DimensionDesc dimensionDesc : cube.getDimensions()) {
        for (TblColRef dimCol : dimensionDesc.getColumnRefs()) {
            if (dimCol.equals(partitionCol)) {
                found = true;
                break;
            }
        }
    }

    if (found == false) {
        context.addResult(ResultLevel.ERROR, "Partition column '" + partitionCol + "' isn't in dimension list.");
        return;
    }

}
Example 14
Source File: HBaseStorage.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
private static TblColRef getPartitionCol(IRealization realization) {
    String modelName = realization.getModel().getName();
    DataModelDesc dataModelDesc = DataModelManager.getInstance(KylinConfig.getInstanceFromEnv())
            .getDataModelDesc(modelName);
    PartitionDesc partitionDesc = dataModelDesc.getPartitionDesc();
    Preconditions.checkArgument(partitionDesc != null, "PartitionDesc for " + realization + " is null!");
    TblColRef partitionColRef = partitionDesc.getPartitionDateColumnRef();
    Preconditions.checkArgument(partitionColRef != null,
            "getPartitionDateColumnRef for " + realization + " is null");
    return partitionColRef;
}
Example 15
Source File: JoinedFlatTable.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
private static void appendWhereStatement(IJoinedFlatTableDesc flatDesc, StringBuilder sql, boolean singleLine, SqlDialect sqlDialect) {
    final String sep = singleLine ? " " : "\n";

    StringBuilder whereBuilder = new StringBuilder();
    whereBuilder.append("WHERE 1=1");

    DataModelDesc model = flatDesc.getDataModel();
    if (StringUtils.isNotEmpty(model.getFilterCondition())) {
        String filterCondition = model.getFilterCondition();
        if (flatDesc.getSegment() != null) {
            JoinedFormatter formatter = new JoinedFormatter(flatDesc);
            filterCondition = formatter.formatSentence(model.getFilterCondition());
        }
        String quotedFilterCondition = quoteIdentifierInSqlExpr(flatDesc, filterCondition, null);
        whereBuilder.append(" AND (").append(quotedFilterCondition).append(") "); // -> filter condition contains special character may cause bug
    }
    if (flatDesc.getSegment() != null) {
        PartitionDesc partDesc = model.getPartitionDesc();
        if (partDesc != null && partDesc.getPartitionDateColumn() != null) {
            SegmentRange segRange = flatDesc.getSegRange();

            if (segRange != null && !segRange.isInfinite()) {
                whereBuilder.append(" AND (");
                String quotedPartitionCond = quoteIdentifierInSqlExpr(flatDesc,
                        partDesc.getPartitionConditionBuilder().buildDateRangeCondition(partDesc, flatDesc.getSegment(), segRange, null), sqlDialect);
                whereBuilder.append(quotedPartitionCond);
                whereBuilder.append(")" + sep);
            }
        }
    }

    sql.append(whereBuilder.toString());
}
Example 16
Source File: JdbcHiveInputBase.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
private static void appendWhereStatement(IJoinedFlatTableDesc flatDesc, StringBuilder sql, boolean singleLine,
        IJdbcMetadata metadata, Map<String, String> metaMap) {
    final String sep = singleLine ? " " : "\n";

    StringBuilder whereBuilder = new StringBuilder();
    whereBuilder.append("WHERE 1=1");

    DataModelDesc model = flatDesc.getDataModel();
    if (StringUtils.isNotEmpty(model.getFilterCondition())) {
        whereBuilder.append(" AND (").append(model.getFilterCondition()).append(") ");
    }

    if (flatDesc.getSegment() != null) {
        PartitionDesc partDesc = model.getPartitionDesc();
        if (partDesc != null && partDesc.getPartitionDateColumn() != null) {
            SegmentRange segRange = flatDesc.getSegRange();

            if (segRange != null && !segRange.isInfinite()) {
                whereBuilder.append(" AND (");
                whereBuilder.append(partDesc.getPartitionConditionBuilder().buildDateRangeCondition(partDesc,
                        flatDesc.getSegment(), segRange,
                        col -> getTableColumnIdentityQuoted(col, metadata, metaMap, true)));
                whereBuilder.append(")");
                whereBuilder.append(sep);
            }
        }
    }
    sql.append(whereBuilder.toString());
}
Example 17
Source File: ModelService.java    From kylin with Apache License 2.0 5 votes vote down vote up 
public void validateModel(String project, DataModelDesc desc) throws IllegalArgumentException {
    String factTableName = desc.getRootFactTableName();
    TableDesc tableDesc = getTableManager().getTableDesc(factTableName, project);
    if ((tableDesc.getSourceType() == ISourceAware.ID_STREAMING || tableDesc.isStreamingTable())
            && (desc.getPartitionDesc() == null || desc.getPartitionDesc().getPartitionDateColumn() == null)) {
        throw new IllegalArgumentException("Must define a partition column.");
    }
}
Example 18
Source File: TableSchemaUpdater.java    From kylin with Apache License 2.0 4 votes vote down vote up 
public static DataModelDesc dealWithMappingForModel(DataModelDesc other,
        Map<String, TableSchemaUpdateMapping> mappings) {
    // For filter condition, not support
    if (!Strings.isNullOrEmpty(other.getFilterCondition())) {
        throw new UnsupportedOperationException("Cannot deal with filter condition " + other.getFilterCondition());
    }

    DataModelDesc copy = DataModelDesc.getCopyOf(other);
    copy.setLastModified(other.getLastModified());

    // mapping for root fact table identity
    TableSchemaUpdateMapping rootMapping = getTableSchemaUpdateMapping(mappings, other.getRootFactTableName());
    if (rootMapping != null) {
        TableDesc rootFactTable = other.getRootFactTable().getTableDesc();
        copy.setRootFactTableName(
                rootMapping.getTableIdentity(rootFactTable.getDatabase(), rootFactTable.getName()));
    }

    // mapping for joins
    JoinTableDesc[] joinTables = other.getJoinTables();
    JoinTableDesc[] joinTablesCopy = new JoinTableDesc[joinTables.length];
    for (int i = 0; i < joinTables.length; i++) {
        JoinTableDesc joinTable = joinTables[i];
        joinTablesCopy[i] = JoinTableDesc.getCopyOf(joinTable);
        String tableIdentity = joinTable.getTable();
        TableSchemaUpdateMapping mapping = getTableSchemaUpdateMapping(mappings, tableIdentity);
        if (mapping != null && mapping.isTableIdentityChanged()) {
            joinTablesCopy[i].setTable(mapping.getTableIdentity(tableIdentity));
        }
    }
    copy.setJoinTables(joinTablesCopy);

    // mapping for partition columns
    PartitionDesc partDesc = other.getPartitionDesc();
    PartitionDesc partCopy = PartitionDesc.getCopyOf(partDesc);
    if (partDesc.getPartitionDateColumnRef() != null) {
        partCopy.setPartitionDateColumn(
                replacePartitionCol(partDesc.getPartitionDateColumnRef().getCanonicalName(), mappings));
    }
    if (partDesc.getPartitionTimeColumnRef() != null) {
        partCopy.setPartitionTimeColumn(
                replacePartitionCol(partDesc.getPartitionTimeColumnRef().getCanonicalName(), mappings));
    }
    copy.setPartitionDesc(partCopy);

    return copy;
}