com.facebook.presto.spi.ColumnMetadata Java Examples

public Optional<List<ColumnMetadata>> getTableColMetadata(String connectorId, String dbName, String tblName)
{
    log.debug("Get list of column metadata of table " + formName(dbName, tblName));
    List<ColumnMetadata> colMetadatas = new ArrayList<>();
    MetaProto.StringListType dataTypeList = metaClient.listColumnsDataType(dbName, tblName);
    MetaProto.StringListType colNameList = metaClient.listColumns(dbName, tblName);
    if (dataTypeList.getIsEmpty() || colNameList.getIsEmpty()) {
        log.warn("No col matches!");
        return Optional.empty();
    }
    for (int i = 0; i < dataTypeList.getStrCount(); i++) {
        String dataType = dataTypeList.getStr(i);
        Type type = getType(dataType);
        ColumnMetadata metadata = new ColumnMetadata(
                colNameList.getStr(i),
                type,
                "",
                false);
        colMetadatas.add(metadata);
    }
    return Optional.of(colMetadatas);
}
 

@Override
public Map<SchemaTableName, List<ColumnMetadata>> listTableColumns(ConnectorSession session, SchemaTablePrefix prefix)
{
    checkNotNull(prefix, "prefix is null");
    log.debug("Called listTableColumns on %s.%s", prefix.getSchemaName(), prefix.getTableName());

    ImmutableMap.Builder<SchemaTableName, List<ColumnMetadata>> columns = ImmutableMap.builder();

    // NOTE: prefix.getTableName or prefix.getSchemaName can be null
    List<SchemaTableName> tableNames;
    if (prefix.getSchemaName() != null && prefix.getTableName() != null) {
        tableNames = ImmutableList.of(new SchemaTableName(prefix.getSchemaName(), prefix.getTableName()));
    }
    else {
        tableNames = listTables(session, (String) null);
    }

    for (SchemaTableName tableName : tableNames) {
        ConnectorTableMetadata tableMetadata = getTableMetadata(tableName);
        if (tableMetadata != null) {
            columns.put(tableName, tableMetadata.getColumns());
        }
    }
    return columns.build();
}
 

private static XContentBuilder getMapping(List<ColumnMetadata> columns)
{
    XContentBuilder mapping = null;
    try {
        mapping = jsonBuilder()
                .startObject().startObject("properties");
        for (ColumnMetadata columnMetadata : columns) {
            String columnName = columnMetadata.getName();
            Type type = columnMetadata.getType();
            if ("@timestamp".equals(columnName)) {    //break @timestamp field
                continue;
            }
            buildFieldType(mapping.startObject(columnName), type).endObject();
        }
        mapping.endObject().endObject();
    }
    catch (IOException e) {
        e.printStackTrace();
    }
    return mapping;
}
 

private static XContentBuilder getMapping(List<ColumnMetadata> columns)
{
    XContentBuilder mapping = null;
    try {
        mapping = jsonBuilder()
                .startObject().startObject("properties");
        for (ColumnMetadata columnMetadata : columns) {
            String columnName = columnMetadata.getName();
            Type type = columnMetadata.getType();
            if ("@timestamp".equals(columnName)) {    //break @timestamp field
                continue;
            }
            buildFieldType(mapping.startObject(columnName), type).endObject();
        }
        mapping.endObject().endObject();
    }
    catch (IOException e) {
        e.printStackTrace();
    }
    return mapping;
}
 

@Override
public Map<SchemaTableName, List<ColumnMetadata>> listTableColumns(ConnectorSession session,
                                                                   SchemaTablePrefix prefix) {
    requireNonNull(prefix, "SchemaTablePrefix is null");

    List<SchemaTableName> tables;
    if (prefix.getSchemaName() == null) {
        tables = listTables(session, Optional.empty());
    } else if (prefix.getTableName() == null) {
        tables = listTables(session, prefix.getSchemaName());
    } else {
        tables = ImmutableList.of(new SchemaTableName(prefix.getSchemaName(), prefix.getTableName()));
    }

    ImmutableMap.Builder<SchemaTableName, List<ColumnMetadata>> columns = ImmutableMap.builder();
    for (SchemaTableName tableName : tables) {
        KuduTableHandle tableHandle = getTableHandle(session, tableName);
        ConnectorTableMetadata tableMetadata = getTableMetadata(tableHandle);
        columns.put(tableName, tableMetadata.getColumns());
    }
    return columns.build();
}
 

public void createTable(ConnectorSession session, ConnectorTableMetadata tableMetadata)
{
    List<ColumnMetadata> columns = tableMetadata.getColumns();
    List<String> columnName = new LinkedList<>();
    List<String> dataType = new LinkedList<>();
    for (ColumnMetadata column : columns) {
        columnName.add(column.getName());
        dataType.add(column.getType().getDisplayName());
    }

    String tblName = tableMetadata.getTable().getTableName();
    String dbName = tableMetadata.getTable().getSchemaName();
    // createTable
    metaClient.createTable(dbName, tblName, session.getUser(),
            "", -1, "",
            -1, columnName, dataType);
}
 

@Override
public Map<SchemaTableName, List<ColumnMetadata>> listTableColumns(ConnectorSession session, SchemaTablePrefix prefix)
{
    requireNonNull(prefix, "prefix is null");

    ImmutableMap.Builder<SchemaTableName, List<ColumnMetadata>> columns = ImmutableMap.builder();

    List<SchemaTableName> tableNames = prefix.getSchemaName() == null ? listTables(session, null) : ImmutableList.of(new SchemaTableName(prefix.getSchemaName(), prefix.getTableName()));

    for (SchemaTableName tableName : tableNames) {
        ConnectorTableMetadata tableMetadata = getTableMetadata(tableName);
        // table can disappear during listing operation
        if (tableMetadata != null) {
            columns.put(tableName, tableMetadata.getColumns());
        }
    }
    return columns.build();
}
 

private static XContentBuilder getMapping(List<ColumnMetadata> columns)
{
    XContentBuilder mapping = null;
    try {
        mapping = jsonBuilder()
                .startObject().startObject("properties");
        for (ColumnMetadata columnMetadata : columns) {
            String columnName = columnMetadata.getName();
            Type type = columnMetadata.getType();
            if ("@timestamp".equals(columnName)) {    //break @timestamp field
                continue;
            }
            buildFieldType(mapping.startObject(columnName), type).endObject();
        }
        mapping.endObject().endObject();
    }
    catch (IOException e) {
        e.printStackTrace();
    }
    return mapping;
}
 

@Override
public KuduTable createTable(ConnectorTableMetadata tableMetadata, boolean ignoreExisting) {
    try {
        SchemaTableName schemeTableName= tableMetadata.getTable();
        String rawName = toRawName(schemeTableName);
        if (ignoreExisting) {
            if (client.tableExists(rawName)) {
                return null;
            }
        }
        if(!schemaExists(schemeTableName.getSchemaName())){
            throw new SchemaNotFoundException(schemeTableName.getSchemaName());
        }
        List<ColumnMetadata> columns = tableMetadata.getColumns();
        Map<String, Object> properties = tableMetadata.getProperties();

        Schema schema = buildSchema(columns, properties);
        CreateTableOptions options = buildCreateTableOptions(schema, properties);
        return client.createTable(rawName, schema, options);
    } catch (KuduException e) {
        throw new PrestoException(GENERIC_INTERNAL_ERROR, e);
    }
}
 

private static void validateLocalityGroups(ConnectorTableMetadata meta)
{
    // Validate any configured locality groups
    Optional<Map<String, Set<String>>> groups = HbaseTableProperties.getLocalityGroups(meta.getProperties());
    if (!groups.isPresent()) {
        return;
    }

    String rowIdColumn = getRowIdColumn(meta);

    // For each locality group
    for (Map.Entry<String, Set<String>> g : groups.get().entrySet()) {
        if (g.getValue().contains(rowIdColumn)) {
            throw new PrestoException(INVALID_TABLE_PROPERTY, "Row ID column cannot be in a locality group");
        }

        // Validate the specified column names exist in the table definition,
        // incrementing a counter for each matching column
        int matchingColumns = 0;
        for (ColumnMetadata column : meta.getColumns()) {
            if (g.getValue().contains(column.getName().toLowerCase(Locale.ENGLISH))) {
                ++matchingColumns;

                // Break out early if all columns are found
                if (matchingColumns == g.getValue().size()) {
                    break;
                }
            }
        }

        // If the number of matched columns does not equal the defined size,
        // then a column was specified that does not exist
        // (or there is a duplicate column in the table DDL, which is also an issue but has been checked before in validateColumns).
        if (matchingColumns != g.getValue().size()) {
            throw new PrestoException(INVALID_TABLE_PROPERTY, "Unknown Presto column defined for locality group " + g.getKey());
        }
    }
}
 

private ConnectorTableMetadata getTableMetadata(KuduTableHandle tableHandle) {
    KuduTable table = tableHandle.getTable(clientSession);
    Schema schema = table.getSchema();

    List<ColumnMetadata> columnsMetaList = schema.getColumns().stream()
            .filter(col -> !col.isKey() || !col.getName().equals(KuduColumnHandle.ROW_ID))
            .map(col -> {
                StringBuilder extra = new StringBuilder();
                if (col.isKey()) {
                    extra.append("key, ");
                } else if (col.isNullable()) {
                    extra.append("nullable, ");
                }
                if (col.getEncoding() != null) {
                    extra.append("encoding=").append(col.getEncoding().name()).append(", ");
                }
                if (col.getCompressionAlgorithm() != null) {
                    extra.append("compression=").append(col.getCompressionAlgorithm().name()).append(", ");
                }
                if (extra.length() > 2) {
                    extra.setLength(extra.length() - 2);
                }
                Type prestoType = TypeHelper.fromKuduColumn(col);
                return new ColumnMetadata(col.getName(), prestoType, null, extra.toString(), false);
            }).collect(toImmutableList());

    Map<String, Object> properties = clientSession.getTableProperties(tableHandle);
    return new ConnectorTableMetadata(tableHandle.getSchemaTableName(), columnsMetaList, properties);
}
 

@Override
public Map<SchemaTableName, List<ColumnMetadata>> listTableColumns(ConnectorSession session, SchemaTablePrefix prefix)
{
    Map<SchemaTableName, List<ColumnMetadata>> tableColumns = new HashMap<>();
    List<SchemaTableName> tableNames = paraflowMetaDataReader.listTables(prefix);
    for (SchemaTableName table : tableNames) {
        List<ColumnMetadata> columnMetadatas = paraflowMetaDataReader.getTableColMetadata(connectorId, table.getSchemaName(),
                table.getTableName()).orElse(new ArrayList<>());
        tableColumns.putIfAbsent(table, columnMetadatas);
    }
    return tableColumns;
}
 

@Override
public ColumnMetadata getColumnMetadata(ConnectorSession session,
                                        ConnectorTableHandle tableHandle, ColumnHandle columnHandle) {
    fromConnectorTableHandle(session, tableHandle);
    KuduColumnHandle kuduColumnHandle = checkType(columnHandle, KuduColumnHandle.class, "columnHandle");
    if (kuduColumnHandle.isVirtualRowId()) {
        return new ColumnMetadata(KuduColumnHandle.ROW_ID, VarbinaryType.VARBINARY, null, true);
    } else {
        return kuduColumnHandle.getColumnMetadata();
    }
}
 

@JsonCreator
public ParaflowTable(
        @JsonProperty("table") ParaflowTableHandle table,
        @JsonProperty("tableLayout") ParaflowTableLayoutHandle tableLayout,
        @JsonProperty("columns") List<ParaflowColumnHandle> columns,
        @JsonProperty("columnMetadatas") List<ColumnMetadata> columnMetadatas)
{
    this.table = requireNonNull(table, "table is null");
    this.tableLayout = requireNonNull(tableLayout, "tableLayout is null");
    this.columns = requireNonNull(columns, "columns is null");
    this.columnMetadatas = requireNonNull(columnMetadatas, "columnMetadas is null");
}
 

@Override
public Map<SchemaTableName, List<ColumnMetadata>> listTableColumns(ConnectorSession session, SchemaTablePrefix prefix)
{
    requireNonNull(prefix, "prefix is null");
    ImmutableMap.Builder<SchemaTableName, List<ColumnMetadata>> columns = ImmutableMap.builder();
    for (SchemaTableName tableName : listTables(session, prefix)) {
        ConnectorTableMetadata tableMetadata = getTableMetadata(tableName);
        // table can disappear during listing operation
        if (tableMetadata != null) {
            columns.put(tableName, tableMetadata.getColumns());
        }
    }
    return columns.build();
}
 

@Override
public void addColumn(SchemaTableName schemaTableName, ColumnMetadata column) {
    try {
        String rawName = toRawName(schemaTableName);
        AlterTableOptions alterOptions = new AlterTableOptions();
        Type type = TypeHelper.toKuduClientType(column.getType());
        alterOptions.addNullableColumn(column.getName(), type);
        client.alterTable(rawName, alterOptions);
    } catch (KuduException e) {
        throw new PrestoException(GENERIC_INTERNAL_ERROR, e);
    }
}
 

private Schema buildSchema(List<ColumnMetadata> columns, Map<String, Object> properties) {
    Optional<Map<String, ColumnDesign>> optColumnDesign = KuduTableProperties.getColumnDesign(properties);

    Map<String, ColumnDesign> columnDesignMap = optColumnDesign.orElse(ImmutableMap.of());

    List<ColumnSchema> kuduColumns = columns.stream()
            .map(columnMetadata -> toColumnSchema(columnMetadata, columnDesignMap))
            .collect(ImmutableList.toImmutableList());
    return new Schema(kuduColumns);
}
 

/**
 * Auto-generates the mapping of Presto column name to Hbase family/qualifier, respecting the locality groups (if any).
 *
 * @param columns Presto columns for the table
 * @param groups Mapping of locality groups to a set of Presto columns, or null if none
 * @return Column mappings
 */
private static Map<String, Pair<String, String>> autoGenerateMapping(List<ColumnMetadata> columns, Optional<Map<String, Set<String>>> groups)
{
    Map<String, Pair<String, String>> mapping = new HashMap<>();
    for (ColumnMetadata column : columns) {
        Optional<String> family = getColumnLocalityGroup(column.getName(), groups);
        mapping.put(column.getName(), Pair.of(family.orElse(column.getName()), column.getName()));
    }
    return mapping;
}
 

private ColumnSchema toColumnSchema(ColumnMetadata columnMetadata, Map<String, ColumnDesign> columnDesignMap) {
    String name = columnMetadata.getName();
    ColumnDesign design = columnDesignMap.getOrDefault(name, ColumnDesign.DEFAULT);
    Type ktype = TypeHelper.toKuduClientType(columnMetadata.getType());
    ColumnSchema.ColumnSchemaBuilder builder = new ColumnSchema.ColumnSchemaBuilder(name, ktype);
    builder.key(design.isKey()).nullable(design.isNullable());
    setEncoding(name, builder, design);
    setCompression(name, builder, design);
    setTypeAttributes(columnMetadata, builder);
    return builder.build();
}
 

private void setTypeAttributes(ColumnMetadata columnMetadata, ColumnSchema.ColumnSchemaBuilder builder) {
    if (columnMetadata.getType() instanceof DecimalType) {
        DecimalType type = (DecimalType) columnMetadata.getType();
        ColumnTypeAttributes attributes = new ColumnTypeAttributes.ColumnTypeAttributesBuilder()
                .precision(type.getPrecision())
                .scale(type.getScale()).build();
        builder.typeAttributes(attributes);
    }
}
 

@Override
public ColumnMetadata getColumnMetadata(
        ConnectorSession session,
        ConnectorTableHandle tableHandle,
        ColumnHandle columnHandle
) {
    convertTableHandle(tableHandle);
    return convertColumnHandle(columnHandle).getColumnMetadata();
}
 

@Override
public Map<SchemaTableName, List<ColumnMetadata>> listTableColumns(ConnectorSession session, SchemaTablePrefix prefix)
{
    requireNonNull(prefix, "prefix is null");
    ImmutableMap.Builder<SchemaTableName, List<ColumnMetadata>> columns = ImmutableMap.builder();
    for (SchemaTableName tableName : listTables(session, prefix)) {
        ConnectorTableMetadata tableMetadata = getTableMetadata(tableName);
        // table can disappear during listing operation
        if (tableMetadata != null) {
            columns.put(tableName, tableMetadata.getColumns());
        }
    }
    return columns.build();
}
 

@Override
public ColumnMetadata getColumnMetadata(ConnectorSession connectorSession, ConnectorTableHandle tableHandle, ColumnHandle columnHandle)
{
    handleResolver.convertTableHandle(tableHandle);
    KinesisColumnHandle kinesisColumnHandle = handleResolver.convertColumnHandle(columnHandle);

    return kinesisColumnHandle.getColumnMetadata();
}
 

public List<ColumnMetadata> getColumns() {
    List<ColumnMetadata> columnMetadataList = new ArrayList<>();
    for (String key : getKubeColumn().keySet()) {
        columnMetadataList.add(getKubeColumn().get(key).getColumnMetadata());
    }
    return columnMetadataList;
}
 

public ColumnMetadata getColumnMetadata() {
    return new ColumnMetadata(name, type);
}
 

private ConnectorTableMetadata getTableMetadata(SchemaTableName tableName)
{
    List<ColumnMetadata> columns = paraflowMetaDataReader.getTableColMetadata(connectorId, tableName.getSchemaName(),
            tableName.getTableName()).orElse(new ArrayList<>());
    return new ConnectorTableMetadata(tableName, columns);
}
 

@Override
public void addColumn(ConnectorSession session, ConnectorTableHandle tableHandle, ColumnMetadata column) {
    KuduTableHandle kuduTableHandle = fromConnectorTableHandle(session, tableHandle);
    clientSession.addColumn(kuduTableHandle.getSchemaTableName(), column);
}
 

ColumnMetadata getColumnMetadata()
{
    return new ColumnMetadata(name, type, null, hidden); // name/type/comment/hidden
}
 

ColumnMetadata getColumnMetadata(boolean hidden)
{
    return new ColumnMetadata(name, type, comment, hidden);
}
 

ColumnMetadata getColumnMetadata()
{
    return new ColumnMetadata(getName(), getType(), getComment(), isHidden());
}