Java Code Examples for org.apache.kylin.metadata.model.TableDesc#setSourceType()

The following examples show how to use org.apache.kylin.metadata.model.TableDesc#setSourceType() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: KylinTableCreator.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
public static TableDesc generateKylinTable(KylinConfig kylinConfig, MetricsSinkDesc sinkDesc, String subject,
                                           List<Pair<String, String>> columns) {
    TableDesc kylinTable = new TableDesc();

    Pair<String, String> tableNameSplits = ActiveReservoirReporter
            .getTableNameSplits(sinkDesc.getTableNameForMetrics(subject));
    kylinTable.setUuid(RandomUtil.randomUUID().toString());
    kylinTable.setDatabase(tableNameSplits.getFirst());
    kylinTable.setName(tableNameSplits.getSecond());
    kylinTable.setTableType(null);
    kylinTable.setLastModified(0L);
    kylinTable.setSourceType(sinkDesc.getSourceType());

    ColumnDesc[] columnDescs = new ColumnDesc[columns.size()];
    for (int i = 0; i < columns.size(); i++) {
        columnDescs[i] = new ColumnDesc();
        Pair<String, String> entry = columns.get(i);
        columnDescs[i].setId(Integer.toString(i + 1));
        columnDescs[i].setName(entry.getFirst());
        columnDescs[i].setDatatype(entry.getSecond());
    }
    kylinTable.setColumns(columnDescs);

    kylinTable.init(kylinConfig, MetricsManager.SYSTEM_PROJECT);

    return kylinTable;
}
Example 2
Source File: JdbcExplorer.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
@Override
public Pair<TableDesc, TableExtDesc> loadTableMetadata(String database, String table, String prj)
        throws SQLException {
    TableDesc tableDesc = new TableDesc();
    tableDesc.setDatabase(database.toUpperCase(Locale.ROOT));
    tableDesc.setName(table.toUpperCase(Locale.ROOT));
    tableDesc.setUuid(RandomUtil.randomUUID().toString());
    tableDesc.setLastModified(0);
    tableDesc.setSourceType(ISourceAware.ID_JDBC);

    Connection con = SqlUtil.getConnection(dbconf);
    DatabaseMetaData dbmd = con.getMetaData();

    try (ResultSet rs = jdbcMetadataDialect.getTable(dbmd, database, table)) {
        String tableType = null;
        while (rs.next()) {
            tableType = rs.getString("TABLE_TYPE");
        }
        if (tableType != null) {
            tableDesc.setTableType(tableType);
        } else {
            throw new RuntimeException(
                    String.format(Locale.ROOT, "table %s not found in schema:%s", table, database));
        }
    }

    try (ResultSet rs = jdbcMetadataDialect.listColumns(dbmd, database, table)) {
        tableDesc.setColumns(extractColumnFromMeta(rs));
    } finally {
        DBUtils.closeQuietly(con);
    }

    TableExtDesc tableExtDesc = new TableExtDesc();
    tableExtDesc.setIdentity(tableDesc.getIdentity());
    tableExtDesc.setUuid(RandomUtil.randomUUID().toString());
    tableExtDesc.setLastModified(0);
    tableExtDesc.init(prj);

    return Pair.newPair(tableDesc, tableExtDesc);
}
Example 3
Source File: TableService.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
public TableDesc generateCsvTableDesc(String tableName, List<String> columnDescList) throws IOException {
    String[] strs = tableName.split("\\.");
    if (strs.length != 2) {
        throw new IllegalArgumentException("Invalid table name + '" + tableName + "'");
    }
    TableDesc tableDesc = new TableDesc();

    tableDesc.setDatabase(strs[0]);
    tableDesc.setName(strs[1]);
    tableDesc.setUuid(RandomUtil.randomUUID().toString());
    tableDesc.setLastModified(0);
    tableDesc.setSourceType(ISourceAware.ID_SPARK);
    List<ColumnDesc> columnDescs = new ArrayList<>();
    int index = 0;

    for (String csvColumnDescStr : columnDescList) {
        index++;
        ColumnDesc columnDesc = new ColumnDesc();
        CsvColumnDesc csvColumnDesc = JsonUtil.readValue(csvColumnDescStr, CsvColumnDesc.class);
        columnDesc.setId("" + index);
        columnDesc.setName((csvColumnDesc).getName());
        columnDesc.setDatatype((csvColumnDesc).getType());
        columnDescs.add(columnDesc);
    }

    tableDesc.setColumns(columnDescs.toArray(new ColumnDesc[columnDescs.size()]));

    return tableDesc;
}
Example 4
Source File: KylinTableCreator.java    From kylin with Apache License 2.0 5 votes vote down vote up 
public static TableDesc generateKylinTable(KylinConfig kylinConfig, MetricsSinkDesc sinkDesc, String subject,
                                           List<Pair<String, String>> columns) {
    TableDesc kylinTable = new TableDesc();

    Pair<String, String> tableNameSplits = ActiveReservoirReporter
            .getTableNameSplits(sinkDesc.getTableNameForMetrics(subject));
    kylinTable.setUuid(RandomUtil.randomUUID().toString());
    kylinTable.setDatabase(tableNameSplits.getFirst());
    kylinTable.setName(tableNameSplits.getSecond());
    kylinTable.setTableType(null);
    kylinTable.setLastModified(0L);
    kylinTable.setSourceType(sinkDesc.getSourceType());

    ColumnDesc[] columnDescs = new ColumnDesc[columns.size()];
    for (int i = 0; i < columns.size(); i++) {
        columnDescs[i] = new ColumnDesc();
        Pair<String, String> entry = columns.get(i);
        columnDescs[i].setId(Integer.toString(i + 1));
        columnDescs[i].setName(entry.getFirst());
        columnDescs[i].setDatatype(entry.getSecond());
    }
    kylinTable.setColumns(columnDescs);

    kylinTable.init(kylinConfig, MetricsManager.SYSTEM_PROJECT);

    return kylinTable;
}
Example 5
Source File: JdbcExplorer.java    From kylin with Apache License 2.0 5 votes vote down vote up 
@Override
public Pair<TableDesc, TableExtDesc> loadTableMetadata(String database, String table, String prj)
        throws SQLException {
    TableDesc tableDesc = new TableDesc();
    tableDesc.setDatabase(database.toUpperCase(Locale.ROOT));
    tableDesc.setName(table.toUpperCase(Locale.ROOT));
    tableDesc.setUuid(RandomUtil.randomUUID().toString());
    tableDesc.setLastModified(0);
    tableDesc.setSourceType(ISourceAware.ID_JDBC);

    Connection con = SqlUtil.getConnection(dbconf);
    DatabaseMetaData dbmd = con.getMetaData();

    try (ResultSet rs = jdbcMetadataDialect.getTable(dbmd, database, table)) {
        String tableType = null;
        while (rs.next()) {
            tableType = rs.getString("TABLE_TYPE");
        }
        if (tableType != null) {
            tableDesc.setTableType(tableType);
        } else {
            throw new RuntimeException(
                    String.format(Locale.ROOT, "table %s not found in schema:%s", table, database));
        }
    }

    try (ResultSet rs = jdbcMetadataDialect.listColumns(dbmd, database, table)) {
        tableDesc.setColumns(extractColumnFromMeta(rs));
    } finally {
        DBUtils.closeQuietly(con);
    }

    TableExtDesc tableExtDesc = new TableExtDesc();
    tableExtDesc.setIdentity(tableDesc.getIdentity());
    tableExtDesc.setUuid(RandomUtil.randomUUID().toString());
    tableExtDesc.setLastModified(0);
    tableExtDesc.init(prj);

    return Pair.newPair(tableDesc, tableExtDesc);
}
Example 6
Source File: JdbcExplorer.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up 
@Override
public Pair<TableDesc, TableExtDesc> loadTableMetadata(String database, String table, String prj)
        throws SQLException {
    TableDesc tableDesc = new TableDesc();
    tableDesc.setDatabase(database.toUpperCase(Locale.ROOT));
    tableDesc.setName(table.toUpperCase(Locale.ROOT));
    tableDesc.setUuid(UUID.randomUUID().toString());
    tableDesc.setLastModified(0);
    tableDesc.setProject(prj);
    tableDesc.setSourceType(JdbcSource.SOURCE_ID);

    try (CachedRowSet tables = dataSource.getTable(database, table)) {
        String tableType = null;
        while (tables.next()) {
            tableType = tables.getString("TABLE_TYPE");
        }
        if (tableType != null) {
            tableDesc.setTableType(tableType);
        } else {
            throw new RuntimeException(String.format(Locale.ROOT, "table %s not found in schema:%s", table, database));
        }
    }

    try (CachedRowSet columns = dataSource.listColumns(database, table)) {
        List<ColumnDesc> columnDescs = new ArrayList<>();

        while (columns.next()) {
            String cname = columns.getString("COLUMN_NAME");
            int type = columns.getInt("DATA_TYPE");
            int csize = columns.getInt("COLUMN_SIZE");
            int digits = columns.getInt("DECIMAL_DIGITS");
            int pos = columns.getInt("ORDINAL_POSITION");
            String remarks = columns.getString("REMARKS");

            ColumnDesc cdesc = new ColumnDesc();
            cdesc.setName(cname.toUpperCase(Locale.ROOT));

            String kylinType = dataSource.toKylinTypeName(type);
            if ("any".equals(kylinType)) {
                String typeName = columns.getString("TYPE_NAME");
                int kylinTypeId = dataSource.toKylinTypeId(typeName, type);
                kylinType = dataSource.toKylinTypeName(kylinTypeId);
            }
            int precision = (SqlUtil.isPrecisionApplicable(kylinType) && csize > 0) ? csize : -1;
            precision = Math.min(precision, KylinConfig.getInstanceFromEnv().getDefaultVarcharPrecision());
            int scale = (SqlUtil.isScaleApplicable(kylinType) && digits > 0) ? digits : -1;

            cdesc.setDatatype(new DataType(kylinType, precision, scale).toString());
            cdesc.setId(String.valueOf(pos));
            cdesc.setComment(remarks);
            columnDescs.add(cdesc);
        }

        tableDesc.setColumns(columnDescs.toArray(new ColumnDesc[columnDescs.size()]));

        TableExtDesc tableExtDesc = new TableExtDesc();
        tableExtDesc.setIdentity(tableDesc.getIdentity());
        tableExtDesc.setUuid(UUID.randomUUID().toString());
        tableExtDesc.setLastModified(0);
        tableExtDesc.init(prj);

        return Pair.newPair(tableDesc, tableExtDesc);
    }
}
Example 7
Source File: JdbcExplorer.java    From kylin with Apache License 2.0 4 votes vote down vote up 
@Override
public Pair<TableDesc, TableExtDesc> loadTableMetadata(String database, String table, String prj)
        throws SQLException {
    TableDesc tableDesc = new TableDesc();
    tableDesc.setDatabase(database.toUpperCase(Locale.ROOT));
    tableDesc.setName(table.toUpperCase(Locale.ROOT));
    tableDesc.setUuid(UUID.randomUUID().toString());
    tableDesc.setLastModified(0);
    tableDesc.setProject(prj);
    tableDesc.setSourceType(JdbcSource.SOURCE_ID);

    try (CachedRowSet tables = dataSource.getTable(database, table)) {
        String tableType = null;
        while (tables.next()) {
            tableType = tables.getString("TABLE_TYPE");
        }
        if (tableType != null) {
            tableDesc.setTableType(tableType);
        } else {
            throw new RuntimeException(String.format(Locale.ROOT, "table %s not found in schema:%s", table, database));
        }
    }

    try (CachedRowSet columns = dataSource.listColumns(database, table)) {
        List<ColumnDesc> columnDescs = new ArrayList<>();

        while (columns.next()) {
            String cname = columns.getString("COLUMN_NAME");
            int type = columns.getInt("DATA_TYPE");
            int csize = columns.getInt("COLUMN_SIZE");
            int digits = columns.getInt("DECIMAL_DIGITS");
            int pos = columns.getInt("ORDINAL_POSITION");
            String remarks = columns.getString("REMARKS");

            ColumnDesc cdesc = new ColumnDesc();
            cdesc.setName(cname.toUpperCase(Locale.ROOT));

            String kylinType = dataSource.toKylinTypeName(type);
            if ("any".equals(kylinType)) {
                String typeName = columns.getString("TYPE_NAME");
                int kylinTypeId = dataSource.toKylinTypeId(typeName, type);
                kylinType = dataSource.toKylinTypeName(kylinTypeId);
            }
            int precision = (SqlUtil.isPrecisionApplicable(kylinType) && csize > 0) ? csize : -1;
            precision = Math.min(precision, KylinConfig.getInstanceFromEnv().getDefaultVarcharPrecision());
            int scale = (SqlUtil.isScaleApplicable(kylinType) && digits > 0) ? digits : -1;

            cdesc.setDatatype(new DataType(kylinType, precision, scale).toString());
            cdesc.setId(String.valueOf(pos));
            cdesc.setComment(remarks);
            columnDescs.add(cdesc);
        }

        tableDesc.setColumns(columnDescs.toArray(new ColumnDesc[columnDescs.size()]));

        TableExtDesc tableExtDesc = new TableExtDesc();
        tableExtDesc.setIdentity(tableDesc.getIdentity());
        tableExtDesc.setUuid(UUID.randomUUID().toString());
        tableExtDesc.setLastModified(0);
        tableExtDesc.init(prj);

        return Pair.newPair(tableDesc, tableExtDesc);
    }
}