Java Code Examples for org.apache.kylin.metadata.model.ColumnDesc#setName()

The following examples show how to use org.apache.kylin.metadata.model.ColumnDesc#setName() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: FilterBaseTest.java    From Kylin with Apache License 2.0 6 votes vote down vote up 
protected List<TblColRef> buildGroups() {
    List<TblColRef> groups = new ArrayList<TblColRef>();

    TableDesc t1 = new TableDesc();
    t1.setName("TEST_KYLIN_FACT");
    t1.setDatabase("DEFAULT");
    ColumnDesc c1 = new ColumnDesc();
    c1.setName("CAL_DT");
    c1.setDatatype("String");
    c1.setTable(t1);
    TblColRef cf1 = new TblColRef(c1);
    groups.add(cf1);

    TableDesc t2 = new TableDesc();
    t2.setName("TEST_CATEGORY_GROUPINGS");
    t2.setDatabase("DEFAULT");
    ColumnDesc c2 = new ColumnDesc();
    c2.setName("META_CATEG_NAME");
    c1.setDatatype("String");
    c2.setTable(t2);
    TblColRef cf2 = new TblColRef(c2);
    groups.add(cf2);

    return groups;
}
Example 2
Source File: CoprocessorRowType.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up 
@Override
public CoprocessorRowType deserialize(ByteBuffer in) {
    int n = BytesUtil.readVInt(in);
    int bodyOffset = BytesUtil.readVInt(in);
    TblColRef[] cols = new TblColRef[n];
    int[] colSizes = new int[n];
    for (int i = 0; i < n; i++) {
        String tableName = BytesUtil.readAsciiString(in);
        String colName = BytesUtil.readAsciiString(in);
        String datatype = BytesUtil.readAsciiString(in);
        TableDesc table = new TableDesc();
        table.setName(tableName);
        ColumnDesc col = new ColumnDesc();
        col.setTable(table);
        col.setName(colName);
        col.setDatatype(datatype);
        col.init(table);
        cols[i] = col.getRef();

        int colSize = BytesUtil.readVInt(in);
        colSizes[i] = colSize;
    }
    return new CoprocessorRowType(cols, colSizes, bodyOffset);
}
Example 3
Source File: StorageTest.java    From Kylin with Apache License 2.0 6 votes vote down vote up 
private List<TblColRef> buildGroups() {
    List<TblColRef> groups = new ArrayList<TblColRef>();

    TableDesc t1 = new TableDesc();
    t1.setName("TEST_KYLIN_FACT");
    t1.setDatabase("DEFAULT");
    ColumnDesc c1 = new ColumnDesc();
    c1.setName("CAL_DT");
    c1.setTable(t1);
    c1.setDatatype("string");
    TblColRef cf1 = new TblColRef(c1);
    groups.add(cf1);

    TableDesc t2 = new TableDesc();
    t2.setName("TEST_CATEGORY_GROUPINGS");
    t2.setDatabase("DEFAULT");
    ColumnDesc c2 = new ColumnDesc();
    c2.setName("META_CATEG_NAME");
    c2.setTable(t2);
    c2.setDatatype("string");
    TblColRef cf2 = new TblColRef(c2);
    groups.add(cf2);

    return groups;
}
Example 4
Source File: CoprocessorRowType.java    From Kylin with Apache License 2.0 6 votes vote down vote up 
@Override
public CoprocessorRowType deserialize(ByteBuffer in) {
    int n = BytesUtil.readVInt(in);
    TblColRef[] cols = new TblColRef[n];
    int[] colSizes = new int[n];
    for (int i = 0; i < n; i++) {
        String tableName = BytesUtil.readAsciiString(in);
        String colName = BytesUtil.readAsciiString(in);
        TableDesc table = new TableDesc();
        table.setName(tableName);
        ColumnDesc col = new ColumnDesc();
        col.setTable(table);
        col.setName(colName);
        cols[i] = new TblColRef(col);

        int colSize = BytesUtil.readVInt(in);
        colSizes[i] = colSize;
    }
    return new CoprocessorRowType(cols, colSizes);
}
Example 5
Source File: ColumnTupleFilter.java    From Kylin with Apache License 2.0 6 votes vote down vote up 
@Override
public void deserialize(byte[] bytes) {
    ColumnDesc column = new ColumnDesc();
    ByteBuffer buffer = ByteBuffer.wrap(bytes);
    String tableName = BytesUtil.readUTFString(buffer);
    if (tableName != null) {
        TableDesc table = new TableDesc();
        table.setName(tableName);
        column.setTable(table);
    }

    column.setName(BytesUtil.readUTFString(buffer));
    column.setDatatype(BytesUtil.readUTFString(buffer));

    this.columnRef = new TblColRef(column);
}
Example 6
Source File: OLAPTable.java    From Kylin with Apache License 2.0 6 votes vote down vote up 
private List<ColumnDesc> listSourceColumns() {
    ProjectManager mgr = ProjectManager.getInstance(olapSchema.getConfig());
    List<ColumnDesc> exposedColumns = Lists.newArrayList(mgr.listExposedColumns(olapSchema.getProjectName(), sourceTable.getIdentity()));

    List<MeasureDesc> countMeasures = mgr.listEffectiveRewriteMeasures(olapSchema.getProjectName(), sourceTable.getIdentity());
    HashSet<String> metFields = new HashSet<String>();
    for (MeasureDesc m : countMeasures) {
        FunctionDesc func = m.getFunction();
        String fieldName = func.getRewriteFieldName();
        if (!metFields.contains(fieldName)) {
            metFields.add(fieldName);
            ColumnDesc fakeCountCol = new ColumnDesc();
            fakeCountCol.setName(fieldName);
            fakeCountCol.setDatatype(func.getSQLType());
            fakeCountCol.setNullable(false);
            fakeCountCol.init(sourceTable);
            exposedColumns.add(fakeCountCol);
        }
    }

    return exposedColumns;
}
Example 7
Source File: CoprocessorRowType.java    From kylin with Apache License 2.0 6 votes vote down vote up 
@Override
public CoprocessorRowType deserialize(ByteBuffer in) {
    int n = BytesUtil.readVInt(in);
    int bodyOffset = BytesUtil.readVInt(in);
    TblColRef[] cols = new TblColRef[n];
    int[] colSizes = new int[n];
    for (int i = 0; i < n; i++) {
        String tableName = BytesUtil.readAsciiString(in);
        String colName = BytesUtil.readAsciiString(in);
        String datatype = BytesUtil.readAsciiString(in);
        TableDesc table = new TableDesc();
        table.setName(tableName);
        ColumnDesc col = new ColumnDesc();
        col.setTable(table);
        col.setName(colName);
        col.setDatatype(datatype);
        col.init(table);
        cols[i] = col.getRef();

        int colSize = BytesUtil.readVInt(in);
        colSizes[i] = colSize;
    }
    return new CoprocessorRowType(cols, colSizes, bodyOffset);
}
Example 8
Source File: FilterSerializeTest.java    From Kylin with Apache License 2.0 5 votes vote down vote up 
@Test
public void testSerialize06() {
    ColumnDesc column = new ColumnDesc();
    column.setName("META_CATEG_NAME");
    TblColRef colRef = new TblColRef(column);
    List<TblColRef> groups = new ArrayList<TblColRef>();
    groups.add(colRef);
    TupleFilter filter = buildCompareFilter(groups, 0);

    byte[] bytes = TupleFilterSerializer.serialize(filter);
    TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes);

    compareFilter(filter, newFilter);
}
Example 9
Source File: HiveMetadataExplorer.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
private ColumnDesc[] extractColumnFromMeta(HiveTableMeta hiveTableMeta) {
    int columnNumber = hiveTableMeta.allColumns.size();
    List<ColumnDesc> columns = new ArrayList<ColumnDesc>(columnNumber);

    for (int i = 0; i < columnNumber; i++) {
        HiveTableMeta.HiveTableColumnMeta field = hiveTableMeta.allColumns.get(i);

        // skip unsupported fields, e.g. map<string, int>
        if (DataType.isKylinSupported(field.dataType)) {
            ColumnDesc cdesc = new ColumnDesc();
            cdesc.setName(field.name.toUpperCase(Locale.ROOT));

            // use "double" in kylin for "float"
            if ("float".equalsIgnoreCase(field.dataType)) {
                cdesc.setDatatype("double");
            } else {
                cdesc.setDatatype(field.dataType);
            }

            cdesc.setId(String.valueOf(i + 1));
            cdesc.setComment(field.comment);
            columns.add(cdesc);
        } else {
            logger.warn("Unsupported data type {}, excluding the field '{}'.", field.dataType, field.name);
        }
    }

    return  columns.toArray(new ColumnDesc[0]);
}
Example 10
Source File: FilterSerializeTest.java    From kylin with Apache License 2.0 5 votes vote down vote up 
@Test
public void testSerialize06() {
    ColumnDesc column = new ColumnDesc();
    column.setName("META_CATEG_NAME");
    TblColRef colRef = column.getRef();
    List<TblColRef> groups = new ArrayList<TblColRef>();
    groups.add(colRef);

    assertFilterSerDe(buildEQCompareFilter(groups, 0));
}
Example 11
Source File: JdbcExplorer.java    From kylin with Apache License 2.0 5 votes vote down vote up 
private ColumnDesc[] extractColumnFromMeta(ResultSet meta) throws SQLException {
    List<ColumnDesc> columns = new ArrayList<>();

    while (meta.next()) {
        String cname = meta.getString("COLUMN_NAME");
        int type = meta.getInt("DATA_TYPE");
        int csize = meta.getInt("COLUMN_SIZE");
        int digits = meta.getInt("DECIMAL_DIGITS");
        int pos = meta.getInt("ORDINAL_POSITION");
        String remarks = meta.getString("REMARKS");

        ColumnDesc cdesc = new ColumnDesc();
        cdesc.setName(cname.toUpperCase(Locale.ROOT));

        String kylinType = SqlUtil.jdbcTypeToKylinDataType(type);
        int precision = (SqlUtil.isPrecisionApplicable(kylinType) && csize > 0) ? csize : -1;
        precision = Math.min(precision, KylinConfig.getInstanceFromEnv().getDefaultVarcharPrecision());
        int scale = (SqlUtil.isScaleApplicable(kylinType) && digits > 0) ? digits : -1;

        cdesc.setDatatype(new DataType(kylinType, precision, scale).toString());
        cdesc.setId(String.valueOf(pos));
        cdesc.setComment(remarks);
        columns.add(cdesc);
    }

    return columns.toArray(new ColumnDesc[columns.size()]);
}
Example 12
Source File: HiveMetadataExplorer.java    From kylin with Apache License 2.0 5 votes vote down vote up 
private ColumnDesc[] extractColumnFromMeta(HiveTableMeta hiveTableMeta) {
    int columnNumber = hiveTableMeta.allColumns.size();
    List<ColumnDesc> columns = new ArrayList<ColumnDesc>(columnNumber);

    for (int i = 0; i < columnNumber; i++) {
        HiveTableMeta.HiveTableColumnMeta field = hiveTableMeta.allColumns.get(i);

        // skip unsupported fields, e.g. map<string, int>
        if (DataType.isKylinSupported(field.dataType)) {
            ColumnDesc cdesc = new ColumnDesc();
            cdesc.setName(field.name.toUpperCase(Locale.ROOT));

            // use "double" in kylin for "float"
            if ("float".equalsIgnoreCase(field.dataType)) {
                cdesc.setDatatype("double");
            } else {
                cdesc.setDatatype(field.dataType);
            }

            cdesc.setId(String.valueOf(i + 1));
            cdesc.setComment(field.comment);
            columns.add(cdesc);
        } else {
            logger.warn("Unsupported data type {}, excluding the field '{}'.", field.dataType, field.name);
        }
    }

    return  columns.toArray(new ColumnDesc[0]);
}
Example 13
Source File: TableService.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
public TableDesc generateCsvTableDesc(String tableName, List<String> columnDescList) throws IOException {
    String[] strs = tableName.split("\\.");
    if (strs.length != 2) {
        throw new IllegalArgumentException("Invalid table name + '" + tableName + "'");
    }
    TableDesc tableDesc = new TableDesc();

    tableDesc.setDatabase(strs[0]);
    tableDesc.setName(strs[1]);
    tableDesc.setUuid(RandomUtil.randomUUID().toString());
    tableDesc.setLastModified(0);
    tableDesc.setSourceType(ISourceAware.ID_SPARK);
    List<ColumnDesc> columnDescs = new ArrayList<>();
    int index = 0;

    for (String csvColumnDescStr : columnDescList) {
        index++;
        ColumnDesc columnDesc = new ColumnDesc();
        CsvColumnDesc csvColumnDesc = JsonUtil.readValue(csvColumnDescStr, CsvColumnDesc.class);
        columnDesc.setId("" + index);
        columnDesc.setName((csvColumnDesc).getName());
        columnDesc.setDatatype((csvColumnDesc).getType());
        columnDescs.add(columnDesc);
    }

    tableDesc.setColumns(columnDescs.toArray(new ColumnDesc[columnDescs.size()]));

    return tableDesc;
}
Example 14
Source File: FilterSerializeTest.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
@Test
public void testSerialize06() {
    ColumnDesc column = new ColumnDesc();
    column.setName("META_CATEG_NAME");
    TblColRef colRef = column.getRef();
    List<TblColRef> groups = new ArrayList<TblColRef>();
    groups.add(colRef);

    assertFilterSerDe(buildEQCompareFilter(groups, 0));
}
Example 15
Source File: JdbcExplorer.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
private ColumnDesc[] extractColumnFromMeta(ResultSet meta) throws SQLException {
    List<ColumnDesc> columns = new ArrayList<>();

    while (meta.next()) {
        String cname = meta.getString("COLUMN_NAME");
        int type = meta.getInt("DATA_TYPE");
        int csize = meta.getInt("COLUMN_SIZE");
        int digits = meta.getInt("DECIMAL_DIGITS");
        int pos = meta.getInt("ORDINAL_POSITION");
        String remarks = meta.getString("REMARKS");

        ColumnDesc cdesc = new ColumnDesc();
        cdesc.setName(cname.toUpperCase(Locale.ROOT));

        String kylinType = SqlUtil.jdbcTypeToKylinDataType(type);
        int precision = (SqlUtil.isPrecisionApplicable(kylinType) && csize > 0) ? csize : -1;
        precision = Math.min(precision, KylinConfig.getInstanceFromEnv().getDefaultVarcharPrecision());
        int scale = (SqlUtil.isScaleApplicable(kylinType) && digits > 0) ? digits : -1;

        cdesc.setDatatype(new DataType(kylinType, precision, scale).toString());
        cdesc.setId(String.valueOf(pos));
        cdesc.setComment(remarks);
        columns.add(cdesc);
    }

    return columns.toArray(new ColumnDesc[columns.size()]);
}
Example 16
Source File: JdbcExplorer.java    From kylin with Apache License 2.0 4 votes vote down vote up 
@Override
public Pair<TableDesc, TableExtDesc> loadTableMetadata(String database, String table, String prj)
        throws SQLException {
    TableDesc tableDesc = new TableDesc();
    tableDesc.setDatabase(database.toUpperCase(Locale.ROOT));
    tableDesc.setName(table.toUpperCase(Locale.ROOT));
    tableDesc.setUuid(UUID.randomUUID().toString());
    tableDesc.setLastModified(0);
    tableDesc.setProject(prj);
    tableDesc.setSourceType(JdbcSource.SOURCE_ID);

    try (CachedRowSet tables = dataSource.getTable(database, table)) {
        String tableType = null;
        while (tables.next()) {
            tableType = tables.getString("TABLE_TYPE");
        }
        if (tableType != null) {
            tableDesc.setTableType(tableType);
        } else {
            throw new RuntimeException(String.format(Locale.ROOT, "table %s not found in schema:%s", table, database));
        }
    }

    try (CachedRowSet columns = dataSource.listColumns(database, table)) {
        List<ColumnDesc> columnDescs = new ArrayList<>();

        while (columns.next()) {
            String cname = columns.getString("COLUMN_NAME");
            int type = columns.getInt("DATA_TYPE");
            int csize = columns.getInt("COLUMN_SIZE");
            int digits = columns.getInt("DECIMAL_DIGITS");
            int pos = columns.getInt("ORDINAL_POSITION");
            String remarks = columns.getString("REMARKS");

            ColumnDesc cdesc = new ColumnDesc();
            cdesc.setName(cname.toUpperCase(Locale.ROOT));

            String kylinType = dataSource.toKylinTypeName(type);
            if ("any".equals(kylinType)) {
                String typeName = columns.getString("TYPE_NAME");
                int kylinTypeId = dataSource.toKylinTypeId(typeName, type);
                kylinType = dataSource.toKylinTypeName(kylinTypeId);
            }
            int precision = (SqlUtil.isPrecisionApplicable(kylinType) && csize > 0) ? csize : -1;
            precision = Math.min(precision, KylinConfig.getInstanceFromEnv().getDefaultVarcharPrecision());
            int scale = (SqlUtil.isScaleApplicable(kylinType) && digits > 0) ? digits : -1;

            cdesc.setDatatype(new DataType(kylinType, precision, scale).toString());
            cdesc.setId(String.valueOf(pos));
            cdesc.setComment(remarks);
            columnDescs.add(cdesc);
        }

        tableDesc.setColumns(columnDescs.toArray(new ColumnDesc[columnDescs.size()]));

        TableExtDesc tableExtDesc = new TableExtDesc();
        tableExtDesc.setIdentity(tableDesc.getIdentity());
        tableExtDesc.setUuid(UUID.randomUUID().toString());
        tableExtDesc.setLastModified(0);
        tableExtDesc.init(prj);

        return Pair.newPair(tableDesc, tableExtDesc);
    }
}
Example 17
Source File: AggregateRegionObserverTest.java    From Kylin with Apache License 2.0 4 votes vote down vote up 
private TblColRef newCol(String name, TableDesc t) {
    ColumnDesc col = new ColumnDesc();
    col.setName(name);
    col.setTable(t);
    return new TblColRef(col);
}
Example 18
Source File: JdbcExplorer.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up 
@Override
public Pair<TableDesc, TableExtDesc> loadTableMetadata(String database, String table, String prj)
        throws SQLException {
    TableDesc tableDesc = new TableDesc();
    tableDesc.setDatabase(database.toUpperCase(Locale.ROOT));
    tableDesc.setName(table.toUpperCase(Locale.ROOT));
    tableDesc.setUuid(UUID.randomUUID().toString());
    tableDesc.setLastModified(0);
    tableDesc.setProject(prj);
    tableDesc.setSourceType(JdbcSource.SOURCE_ID);

    try (CachedRowSet tables = dataSource.getTable(database, table)) {
        String tableType = null;
        while (tables.next()) {
            tableType = tables.getString("TABLE_TYPE");
        }
        if (tableType != null) {
            tableDesc.setTableType(tableType);
        } else {
            throw new RuntimeException(String.format(Locale.ROOT, "table %s not found in schema:%s", table, database));
        }
    }

    try (CachedRowSet columns = dataSource.listColumns(database, table)) {
        List<ColumnDesc> columnDescs = new ArrayList<>();

        while (columns.next()) {
            String cname = columns.getString("COLUMN_NAME");
            int type = columns.getInt("DATA_TYPE");
            int csize = columns.getInt("COLUMN_SIZE");
            int digits = columns.getInt("DECIMAL_DIGITS");
            int pos = columns.getInt("ORDINAL_POSITION");
            String remarks = columns.getString("REMARKS");

            ColumnDesc cdesc = new ColumnDesc();
            cdesc.setName(cname.toUpperCase(Locale.ROOT));

            String kylinType = dataSource.toKylinTypeName(type);
            if ("any".equals(kylinType)) {
                String typeName = columns.getString("TYPE_NAME");
                int kylinTypeId = dataSource.toKylinTypeId(typeName, type);
                kylinType = dataSource.toKylinTypeName(kylinTypeId);
            }
            int precision = (SqlUtil.isPrecisionApplicable(kylinType) && csize > 0) ? csize : -1;
            precision = Math.min(precision, KylinConfig.getInstanceFromEnv().getDefaultVarcharPrecision());
            int scale = (SqlUtil.isScaleApplicable(kylinType) && digits > 0) ? digits : -1;

            cdesc.setDatatype(new DataType(kylinType, precision, scale).toString());
            cdesc.setId(String.valueOf(pos));
            cdesc.setComment(remarks);
            columnDescs.add(cdesc);
        }

        tableDesc.setColumns(columnDescs.toArray(new ColumnDesc[columnDescs.size()]));

        TableExtDesc tableExtDesc = new TableExtDesc();
        tableExtDesc.setIdentity(tableDesc.getIdentity());
        tableExtDesc.setUuid(UUID.randomUUID().toString());
        tableExtDesc.setLastModified(0);
        tableExtDesc.init(prj);

        return Pair.newPair(tableDesc, tableExtDesc);
    }
}