Java Code Examples for org.apache.kylin.metadata.model.TableDesc#findColumnByName()

The following examples show how to use org.apache.kylin.metadata.model.TableDesc#findColumnByName() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TableSchemaUpdateChecker.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up 
/**
 * check whether all columns used in `cube` has compatible schema in current hive schema denoted by `fieldsMap`.
 * @param cube cube to check, must use `table` in its model
 * @param origTable kylin's table metadata
 * @param newTable current hive schema of `table`
 * @return columns in origTable that can't be found in newTable
 */
private List<String> checkAllColumnsInCube(CubeInstance cube, TableDesc origTable, TableDesc newTable) {
    Set<ColumnDesc> usedColumns = Sets.newHashSet();
    for (TblColRef col : cube.getAllColumns()) {
        usedColumns.add(col.getColumnDesc());
    }

    List<String> violateColumns = Lists.newArrayList();
    for (ColumnDesc column : origTable.getColumns()) {
        if (!column.isComputedColumn() && usedColumns.contains(column)) {
            ColumnDesc newCol = newTable.findColumnByName(column.getName());
            if (newCol == null || !isColumnCompatible(column, newCol)) {
                violateColumns.add(column.getName());
            }
        }
    }
    return violateColumns;
}
Example 2
Source File: TableSchemaUpdateChecker.java    From kylin with Apache License 2.0 6 votes vote down vote up 
/**
 * check whether all columns used in `cube` has compatible schema in current hive schema denoted by `fieldsMap`.
 * @param cube cube to check, must use `table` in its model
 * @param origTable kylin's table metadata
 * @param newTable current hive schema of `table`
 * @return columns in origTable that can't be found in newTable
 */
private List<String> checkAllColumnsInCube(CubeInstance cube, TableDesc origTable, TableDesc newTable) {
    Set<ColumnDesc> usedColumns = Sets.newHashSet();
    for (TblColRef col : cube.getAllColumns()) {
        usedColumns.add(col.getColumnDesc());
    }

    List<String> violateColumns = Lists.newArrayList();
    for (ColumnDesc column : origTable.getColumns()) {
        if (!column.isComputedColumn() && usedColumns.contains(column)) {
            ColumnDesc newCol = newTable.findColumnByName(column.getName());
            if (newCol == null || !isColumnCompatible(column, newCol)) {
                violateColumns.add(column.getName());
            }
        }
    }
    return violateColumns;
}
Example 3
Source File: ProjectL2Cache.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
private boolean sanityCheck(ProjectCache prjCache, IRealization realization) {
    if (realization == null)
        return false;

    TableMetadataManager metaMgr = mgr.getTableManager();

    Set<TblColRef> allColumns = realization.getAllColumns();
    if (allColumns == null || allColumns.isEmpty()) {
        logger.error("Realization '" + realization.getCanonicalName() + "' does not report any columns");
        return false;
    }

    for (TblColRef col : allColumns) {
        TableDesc table = metaMgr.getTableDesc(col.getTable(), prjCache.project);
        if (table == null) {
            logger.error("Realization '" + realization.getCanonicalName() + "' reports column '" + col.getCanonicalName() + "', but its table is not found by MetadataManager");
            return false;
        }

        if (!col.getColumnDesc().isComputedColumn()) {
            ColumnDesc foundCol = table.findColumnByName(col.getName());
            if (col.getColumnDesc().equals(foundCol) == false) {
                logger.error("Realization '" + realization.getCanonicalName() + "' reports column '" + col.getCanonicalName() + "', but it is not equal to '" + foundCol + "' according to MetadataManager");
                return false;
            }
        } else {
            //computed column may not exit here
        }

        // auto-define table required by realization for some legacy test case
        if (prjCache.tables.get(table.getIdentity()) == null) {
            prjCache.tables.put(table.getIdentity(), new TableCache(table));
            logger.warn("Realization '" + realization.getCanonicalName() + "' reports column '" + col.getCanonicalName() + "' whose table is not defined in project '" + prjCache.project + "'");
        }
    }

    return true;
}
Example 4
Source File: TableSchemaUpdateChecker.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
private List<String> checkAllColumnsInFactTable(DataModelDesc usedModel, TableDesc factTable, TableDesc newTableDesc) {
    List<String> violateColumns = Lists.newArrayList();

    for (ColumnDesc column : findUsedColumnsInFactTable(usedModel, factTable)) {
        if (!column.isComputedColumn()) {
            ColumnDesc newCol = newTableDesc.findColumnByName(column.getName());
            if (newCol == null || !isColumnCompatible(column, newCol)) {
                violateColumns.add(column.getName());
            }
        }
    }
    return violateColumns;
}
Example 5
Source File: ProjectL2Cache.java    From kylin with Apache License 2.0 5 votes vote down vote up 
private boolean sanityCheck(ProjectCache prjCache, IRealization realization) {
    if (realization == null)
        return false;

    TableMetadataManager metaMgr = mgr.getTableManager();

    Set<TblColRef> allColumns = realization.getAllColumns();
    if (allColumns == null || allColumns.isEmpty()) {
        logger.error("Realization '" + realization.getCanonicalName() + "' does not report any columns");
        return false;
    }

    for (TblColRef col : allColumns) {
        TableDesc table = metaMgr.getTableDesc(col.getTable(), prjCache.project);
        if (table == null) {
            logger.error("Realization '" + realization.getCanonicalName() + "' reports column '" + col.getCanonicalName() + "', but its table is not found by MetadataManager");
            return false;
        }

        if (!col.getColumnDesc().isComputedColumn()) {
            ColumnDesc foundCol = table.findColumnByName(col.getName());
            if (col.getColumnDesc().equals(foundCol) == false) {
                logger.error("Realization '" + realization.getCanonicalName() + "' reports column '" + col.getCanonicalName() + "', but it is not equal to '" + foundCol + "' according to MetadataManager");
                return false;
            }
        } else {
            //computed column may not exit here
        }

        // auto-define table required by realization for some legacy test case
        if (prjCache.tables.get(table.getIdentity()) == null) {
            prjCache.tables.put(table.getIdentity(), new TableCache(table));
            logger.warn("Realization '" + realization.getCanonicalName() + "' reports column '" + col.getCanonicalName() + "' whose table is not defined in project '" + prjCache.project + "'");
        }
    }

    return true;
}
Example 6
Source File: TableSchemaUpdateChecker.java    From kylin with Apache License 2.0 5 votes vote down vote up 
private List<String> checkAllColumnsInFactTable(DataModelDesc usedModel, TableDesc factTable,
        TableDesc newTableDesc) {
    List<String> violateColumns = Lists.newArrayList();

    for (ColumnDesc column : findUsedColumnsInFactTable(usedModel, factTable)) {
        if (!column.isComputedColumn()) {
            ColumnDesc newCol = newTableDesc.findColumnByName(column.getName());
            if (newCol == null) {
                violateColumns.add(column.getName());
            }
        }
    }
    return violateColumns;
}
Example 7
Source File: CubeDesc.java    From Kylin with Apache License 2.0 5 votes vote down vote up 
private TblColRef initDimensionColRef(TableDesc table, String colName) {
    ColumnDesc col = table.findColumnByName(colName);
    if (col == null)
        throw new IllegalArgumentException("No column '" + colName + "' found in table " + table);

    TblColRef ref = new TblColRef(col);
    return initDimensionColRef(ref);
}
Example 8
Source File: CubeDesc.java    From Kylin with Apache License 2.0 5 votes vote down vote up 
private void initMeasureColumns(Map<String, TableDesc> tables) {
    if (measures == null || measures.isEmpty()) {
        return;
    }

    TableDesc factTable = tables.get(getFactTable());
    for (MeasureDesc m : measures) {
        m.setName(m.getName().toUpperCase());

        if (m.getDependentMeasureRef() != null) {
            m.setDependentMeasureRef(m.getDependentMeasureRef().toUpperCase());
        }

        FunctionDesc f = m.getFunction();
        f.setExpression(f.getExpression().toUpperCase());
        f.setReturnDataType(DataType.getInstance(f.getReturnType()));

        ParameterDesc p = f.getParameter();
        p.normalizeColumnValue();

        if (p.isColumnType()) {
            ArrayList<TblColRef> colRefs = Lists.newArrayList();
            for (String cName : p.getValue().split("\\s*,\\s*")) {
                ColumnDesc sourceColumn = factTable.findColumnByName(cName);
                TblColRef colRef = new TblColRef(sourceColumn);
                colRefs.add(colRef);
                allColumns.add(colRef);
            }
            if (colRefs.isEmpty() == false)
                p.setColRefs(colRefs);
        }
    }
}
Example 9
Source File: CubeDesc.java    From Kylin with Apache License 2.0 5 votes vote down vote up 
private TblColRef initDimensionColRef(DimensionDesc dim, String colName) {
    TableDesc table = dim.getTableDesc();
    ColumnDesc col = table.findColumnByName(colName);
    if (col == null)
        throw new IllegalArgumentException("No column '" + colName + "' found in table " + table);
    
    TblColRef ref = new TblColRef(col);
    return initDimensionColRef(ref);
}
Example 10
Source File: CubeDesc.java    From Kylin with Apache License 2.0 5 votes vote down vote up 
private void initMeasureColumns(Map<String, TableDesc> tables) {
    if (measures == null || measures.isEmpty()) {
        return;
    }

    TableDesc factTable = tables.get(getFactTable());
    for (MeasureDesc m : measures) {
        m.setName(m.getName().toUpperCase());

        if (m.getDependentMeasureRef() != null) {
            m.setDependentMeasureRef(m.getDependentMeasureRef().toUpperCase());
        }
        
        FunctionDesc f = m.getFunction();
        f.setExpression(f.getExpression().toUpperCase());
        f.setReturnDataType(DataType.getInstance(f.getReturnType()));

        ParameterDesc p = f.getParameter();
        p.normalizeColumnValue();

        if (p.isColumnType()) {
            ArrayList<TblColRef> colRefs = Lists.newArrayList();
            for (String cName : p.getValue().split("\\s*,\\s*")) {
                ColumnDesc sourceColumn = factTable.findColumnByName(cName);
                TblColRef colRef = new TblColRef(sourceColumn);
                colRefs.add(colRef);
                allColumns.add(colRef);
            }
            if (colRefs.isEmpty() == false)
                p.setColRefs(colRefs);
        }
        
        // verify holistic count distinct as a dependent measure
        if (m.isHolisticCountDistinct() && StringUtils.isBlank(m.getDependentMeasureRef())) {
            throw new IllegalStateException(m + " is a holistic count distinct but it has no DependentMeasureRef defined!");
        }
    }
}
Example 11
Source File: TableSchemaUpdateChecker.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up 
private ColumnDesc mustGetColumnDesc(TableDesc factTable, String columnName) {
    ColumnDesc columnDesc = factTable.findColumnByName(columnName);
    Preconditions.checkNotNull(columnDesc,
            format(Locale.ROOT, "Can't find column %s in current fact table %s.", columnName, factTable.getIdentity()));
    return columnDesc;
}
Example 12
Source File: TableSchemaUpdateChecker.java    From kylin with Apache License 2.0 4 votes vote down vote up 
private ColumnDesc mustGetColumnDesc(TableDesc factTable, String columnName) {
    ColumnDesc columnDesc = factTable.findColumnByName(columnName);
    Preconditions.checkNotNull(columnDesc,
            format(Locale.ROOT, "Can't find column %s in current fact table %s.", columnName, factTable.getIdentity()));
    return columnDesc;
}