com.facebook.presto.spi.ConnectorTableMetadata Java Examples

@Override
public Map<SchemaTableName, List<ColumnMetadata>> listTableColumns(ConnectorSession session, SchemaTablePrefix prefix)
{
    checkNotNull(prefix, "prefix is null");
    log.debug("Called listTableColumns on %s.%s", prefix.getSchemaName(), prefix.getTableName());

    ImmutableMap.Builder<SchemaTableName, List<ColumnMetadata>> columns = ImmutableMap.builder();

    // NOTE: prefix.getTableName or prefix.getSchemaName can be null
    List<SchemaTableName> tableNames;
    if (prefix.getSchemaName() != null && prefix.getTableName() != null) {
        tableNames = ImmutableList.of(new SchemaTableName(prefix.getSchemaName(), prefix.getTableName()));
    }
    else {
        tableNames = listTables(session, (String) null);
    }

    for (SchemaTableName tableName : tableNames) {
        ConnectorTableMetadata tableMetadata = getTableMetadata(tableName);
        if (tableMetadata != null) {
            columns.put(tableName, tableMetadata.getColumns());
        }
    }
    return columns.build();
}
 

@Override
public KuduTable createTable(ConnectorTableMetadata tableMetadata, boolean ignoreExisting) {
    try {
        SchemaTableName schemeTableName= tableMetadata.getTable();
        String rawName = toRawName(schemeTableName);
        if (ignoreExisting) {
            if (client.tableExists(rawName)) {
                return null;
            }
        }
        if(!schemaExists(schemeTableName.getSchemaName())){
            throw new SchemaNotFoundException(schemeTableName.getSchemaName());
        }
        List<ColumnMetadata> columns = tableMetadata.getColumns();
        Map<String, Object> properties = tableMetadata.getProperties();

        Schema schema = buildSchema(columns, properties);
        CreateTableOptions options = buildCreateTableOptions(schema, properties);
        return client.createTable(rawName, schema, options);
    } catch (KuduException e) {
        throw new PrestoException(GENERIC_INTERNAL_ERROR, e);
    }
}
 

@Override
public Map<SchemaTableName, List<ColumnMetadata>> listTableColumns(ConnectorSession session,
                                                                   SchemaTablePrefix prefix) {
    requireNonNull(prefix, "SchemaTablePrefix is null");

    List<SchemaTableName> tables;
    if (prefix.getSchemaName() == null) {
        tables = listTables(session, Optional.empty());
    } else if (prefix.getTableName() == null) {
        tables = listTables(session, prefix.getSchemaName());
    } else {
        tables = ImmutableList.of(new SchemaTableName(prefix.getSchemaName(), prefix.getTableName()));
    }

    ImmutableMap.Builder<SchemaTableName, List<ColumnMetadata>> columns = ImmutableMap.builder();
    for (SchemaTableName tableName : tables) {
        KuduTableHandle tableHandle = getTableHandle(session, tableName);
        ConnectorTableMetadata tableMetadata = getTableMetadata(tableHandle);
        columns.put(tableName, tableMetadata.getColumns());
    }
    return columns.build();
}
 

@Override
public ConnectorOutputTableHandle beginCreateTable(ConnectorSession session, ConnectorTableMetadata tableMetadata, Optional<ConnectorNewTableLayout> layout)
{
    checkNoRollback();   //采用乐观锁 设计

    SchemaTableName tableName = tableMetadata.getTable();
    client.createTable(tableMetadata);

    //--- es column is sort ----
    List<ElasticsearchColumnHandle> columns = tableMetadata.getColumns().stream()
            .map(columnMetadata -> new ElasticsearchColumnHandle(
                    columnMetadata.getName(),
                    columnMetadata.getType(),
                    columnMetadata.getComment(),
                    true,
                    columnMetadata.isHidden())).collect(Collectors.toList());
    ElasticsearchOutputTableHandle handle = new ElasticsearchOutputTableHandle(
            connectorId,
            tableName.getSchemaName(),
            tableName.getTableName(),
            columns);

    setRollback(() -> client.dropTable(tableMetadata.getTable()));  //回滚操作

    return handle;
}
 

@Override
public Map<SchemaTableName, List<ColumnMetadata>> listTableColumns(ConnectorSession session, SchemaTablePrefix prefix)
{
    requireNonNull(prefix, "prefix is null");

    ImmutableMap.Builder<SchemaTableName, List<ColumnMetadata>> columns = ImmutableMap.builder();

    List<SchemaTableName> tableNames = prefix.getSchemaName() == null ? listTables(session, null) : ImmutableList.of(new SchemaTableName(prefix.getSchemaName(), prefix.getTableName()));

    for (SchemaTableName tableName : tableNames) {
        ConnectorTableMetadata tableMetadata = getTableMetadata(tableName);
        // table can disappear during listing operation
        if (tableMetadata != null) {
            columns.put(tableName, tableMetadata.getColumns());
        }
    }
    return columns.build();
}
 

public void createTable(ConnectorSession session, ConnectorTableMetadata tableMetadata)
{
    List<ColumnMetadata> columns = tableMetadata.getColumns();
    List<String> columnName = new LinkedList<>();
    List<String> dataType = new LinkedList<>();
    for (ColumnMetadata column : columns) {
        columnName.add(column.getName());
        dataType.add(column.getType().getDisplayName());
    }

    String tblName = tableMetadata.getTable().getTableName();
    String dbName = tableMetadata.getTable().getSchemaName();
    // createTable
    metaClient.createTable(dbName, tblName, session.getUser(),
            "", -1, "",
            -1, columnName, dataType);
}
 

private ConnectorTableMetadata getTableMetadata(SchemaTableName tableName)
{
    if (!client.getSchemaNames().contains(tableName.getSchemaName())) {
        return null;
    }

    // Need to validate that SchemaTableName is a table
    if (!this.listViews(tableName.getSchemaName()).contains(tableName)) {
        HbaseTable table = client.getTable(tableName);
        if (table == null) {
            return null;
        }

        return new ConnectorTableMetadata(tableName, table.getColumnsMetadata());
    }

    return null;
}
 

@Override
public ConnectorOutputTableHandle beginCreateTable(ConnectorSession session, ConnectorTableMetadata tableMetadata, Optional<ConnectorNewTableLayout> layout)
{
    checkNoRollback();   //采用乐观锁 设计

    SchemaTableName tableName = tableMetadata.getTable();
    HbaseTable table = client.createTable(tableMetadata);

    HbaseTableHandle handle = new HbaseTableHandle(
            connectorId,
            tableName.getSchemaName(),
            tableName.getTableName(),
            table.getRowId(),
            table.isExternal(),
            table.getScanAuthorizations());

    setRollback(() -> client.dropTable(table));  //回滚操作

    return handle;
}
 

@Override
public void createTable(ConnectorTableMetadata tableMetadata)
{
    XContentBuilder mapping = getMapping(tableMetadata.getColumns());
    String index = tableMetadata.getTable().getTableName();
    try {
        //TODO: default type value is presto
        client.admin().indices().prepareCreate(index)
                .addMapping("presto", mapping)
                .execute().actionGet();
    }
    catch (MapperParsingException e) {
        throw new PrestoException(ES_MAPPING_ERROR, "Failed create index:" + index, e);
    }
}
 

@Override
public ConnectorTableMetadata getTableMetadata(ConnectorSession connectorSession, ConnectorTableHandle tableHandle)
{
    KinesisTableHandle kinesisTableHandle = handleResolver.convertTableHandle(tableHandle);
    log.debug("Called getTableMetadata on %s.%s", kinesisTableHandle.getSchemaName(), kinesisTableHandle.getTableName());
    return getTableMetadata(kinesisTableHandle.toSchemaTableName());
}
 

private ConnectorTableMetadata getTableMetadata(KuduTableHandle tableHandle) {
    KuduTable table = tableHandle.getTable(clientSession);
    Schema schema = table.getSchema();

    List<ColumnMetadata> columnsMetaList = schema.getColumns().stream()
            .filter(col -> !col.isKey() || !col.getName().equals(KuduColumnHandle.ROW_ID))
            .map(col -> {
                StringBuilder extra = new StringBuilder();
                if (col.isKey()) {
                    extra.append("key, ");
                } else if (col.isNullable()) {
                    extra.append("nullable, ");
                }
                if (col.getEncoding() != null) {
                    extra.append("encoding=").append(col.getEncoding().name()).append(", ");
                }
                if (col.getCompressionAlgorithm() != null) {
                    extra.append("compression=").append(col.getCompressionAlgorithm().name()).append(", ");
                }
                if (extra.length() > 2) {
                    extra.setLength(extra.length() - 2);
                }
                Type prestoType = TypeHelper.fromKuduColumn(col);
                return new ColumnMetadata(col.getName(), prestoType, null, extra.toString(), false);
            }).collect(toImmutableList());

    Map<String, Object> properties = clientSession.getTableProperties(tableHandle);
    return new ConnectorTableMetadata(tableHandle.getSchemaTableName(), columnsMetaList, properties);
}
 

@Override
public ConnectorTableMetadata getTableMetadata(ConnectorSession session, ConnectorTableHandle table)
{
    ParaflowTableHandle paraflowTable = (ParaflowTableHandle) table;
    checkArgument(paraflowTable.getConnectorId().equals(connectorId), "tableHandle is not for this connector");
    SchemaTableName tableName = new SchemaTableName(paraflowTable.getSchemaName(), paraflowTable.getTableName());
    return getTableMetadata(tableName);
}
 

@Override
public void createTable(ConnectorTableMetadata tableMetadata)
{
    XContentBuilder mapping = getMapping(tableMetadata.getColumns());
    String index = tableMetadata.getTable().getTableName();
    try {
        //TODO: default type value is presto
        client.admin().indices().prepareCreate(index)
                .addMapping("presto", mapping)
                .execute().actionGet();
    }
    catch (MapperParsingException e) {
        throw new PrestoException(ES_MAPPING_ERROR, "Failed create index:" + index, e);
    }
}
 

@Override
public Map<SchemaTableName, List<ColumnMetadata>> listTableColumns(ConnectorSession session, SchemaTablePrefix prefix)
{
    requireNonNull(prefix, "prefix is null");
    ImmutableMap.Builder<SchemaTableName, List<ColumnMetadata>> columns = ImmutableMap.builder();
    for (SchemaTableName tableName : listTables(session, prefix)) {
        ConnectorTableMetadata tableMetadata = getTableMetadata(tableName);
        // table can disappear during listing operation
        if (tableMetadata != null) {
            columns.put(tableName, tableMetadata.getColumns());
        }
    }
    return columns.build();
}
 

@Override
public ConnectorTableMetadata getTableMetadata(ConnectorSession session, ConnectorTableHandle table)
{
    HbaseTableHandle handle = (HbaseTableHandle) table;
    checkArgument(handle.getConnectorId().equals(connectorId), "table is not for this connector");
    SchemaTableName tableName = new SchemaTableName(handle.getSchema(), handle.getTable());
    ConnectorTableMetadata metadata = getTableMetadata(tableName);
    if (metadata == null) {
        throw new TableNotFoundException(tableName);
    }
    return metadata;
}
 

private void validateInternalTable(ConnectorTableMetadata meta)
{
    String table = HbaseTable.getFullTableName(meta.getTable());

    if (tableManager.exists(table)) {
        throw new PrestoException(HBASE_TABLE_EXISTS, "Cannot create internal table when an Hbase table already exists");
    }
}
 

private static void validateLocalityGroups(ConnectorTableMetadata meta)
{
    // Validate any configured locality groups
    Optional<Map<String, Set<String>>> groups = HbaseTableProperties.getLocalityGroups(meta.getProperties());
    if (!groups.isPresent()) {
        return;
    }

    String rowIdColumn = getRowIdColumn(meta);

    // For each locality group
    for (Map.Entry<String, Set<String>> g : groups.get().entrySet()) {
        if (g.getValue().contains(rowIdColumn)) {
            throw new PrestoException(INVALID_TABLE_PROPERTY, "Row ID column cannot be in a locality group");
        }

        // Validate the specified column names exist in the table definition,
        // incrementing a counter for each matching column
        int matchingColumns = 0;
        for (ColumnMetadata column : meta.getColumns()) {
            if (g.getValue().contains(column.getName().toLowerCase(Locale.ENGLISH))) {
                ++matchingColumns;

                // Break out early if all columns are found
                if (matchingColumns == g.getValue().size()) {
                    break;
                }
            }
        }

        // If the number of matched columns does not equal the defined size,
        // then a column was specified that does not exist
        // (or there is a duplicate column in the table DDL, which is also an issue but has been checked before in validateColumns).
        if (matchingColumns != g.getValue().size()) {
            throw new PrestoException(INVALID_TABLE_PROPERTY, "Unknown Presto column defined for locality group " + g.getKey());
        }
    }
}
 

/**
 * Validates the given metadata for a series of conditions to ensure the table is well-formed.
 *
 * @param meta Table metadata
 */
private void validateCreateTable(ConnectorTableMetadata meta)
{
    validateColumns(meta);
    validateLocalityGroups(meta);
    if (!HbaseTableProperties.isExternal(meta.getProperties())) {
        validateInternalTable(meta);
    }
}
 

@Override
public void createTable(ConnectorTableMetadata tableMetadata)
{
    XContentBuilder mapping = getMapping(tableMetadata.getColumns());
    String index = tableMetadata.getTable().getTableName();
    try {
        //TODO: default type value is presto
        client.admin().indices().prepareCreate(index)
                .addMapping("presto", mapping)
                .execute().actionGet();
    }
    catch (MapperParsingException e) {
        throw new PrestoException(ES_MAPPING_ERROR, "Failed create index:" + index, e);
    }
}
 

@Override
public ConnectorTableMetadata getTableMetadata(ConnectorSession session, ConnectorTableHandle table)
{
    ElasticsearchTableHandle handle = (ElasticsearchTableHandle) table;
    checkArgument(handle.getConnectorId().equals(connectorId), "table is not for this connector");
    SchemaTableName tableName = new SchemaTableName(handle.getSchemaName(), handle.getTableName());
    ConnectorTableMetadata metadata = getTableMetadata(tableName);
    if (metadata == null) {
        throw new TableNotFoundException(tableName);
    }
    return metadata;
}
 

@Override
public Map<SchemaTableName, List<ColumnMetadata>> listTableColumns(ConnectorSession session, SchemaTablePrefix prefix)
{
    requireNonNull(prefix, "prefix is null");
    ImmutableMap.Builder<SchemaTableName, List<ColumnMetadata>> columns = ImmutableMap.builder();
    for (SchemaTableName tableName : listTables(session, prefix)) {
        ConnectorTableMetadata tableMetadata = getTableMetadata(tableName);
        // table can disappear during listing operation
        if (tableMetadata != null) {
            columns.put(tableName, tableMetadata.getColumns());
        }
    }
    return columns.build();
}
 

private ConnectorTableMetadata getTableMetadata(SchemaTableName tableName)
{
    // Need to validate that SchemaTableName is a table
    ElasticsearchTable table = client.getTable(tableName);
    if (table == null) {
        return null;
    }

    return new ConnectorTableMetadata(tableName, table.getColumnsMetadata());
}
 

@Override
public void createTable(ConnectorSession session, ConnectorTableMetadata tableMetadata, boolean ignoreExisting)
{
    client.createTable(tableMetadata);
}
 

@Test
public void testRelatedObjects()
{
    // Metadata has a handle to the supplier, ensure it can use it correctly
    KinesisMetadata meta = injector.getInstance(KinesisMetadata.class);
    assertNotNull(meta);

    Map<SchemaTableName, KinesisStreamDescription> tblMap = meta.getDefinedTables();
    SchemaTableName tblName = new SchemaTableName("prod", "test_table");
    KinesisStreamDescription desc = tblMap.get(tblName);
    assertNotNull(desc);
    assertEquals(desc.getSchemaName(), "prod");
    assertEquals(desc.getTableName(), "test_table");
    assertEquals(desc.getStreamName(), "test_kinesis_stream");

    List<String> schemas = meta.listSchemaNames(null);
    assertEquals(schemas.size(), 1);
    assertEquals(schemas.get(0), "prod");

    KinesisTableHandle tblHandle = meta.getTableHandle(null, tblName);
    assertNotNull(tblHandle);
    assertEquals(tblHandle.getSchemaName(), "prod");
    assertEquals(tblHandle.getTableName(), "test_table");
    assertEquals(tblHandle.getStreamName(), "test_kinesis_stream");
    assertEquals(tblHandle.getMessageDataFormat(), "json");

    ConnectorTableMetadata tblMeta = meta.getTableMetadata(null, tblHandle);
    assertNotNull(tblMeta);
    assertEquals(tblMeta.getTable().getSchemaName(), "prod");
    assertEquals(tblMeta.getTable().getTableName(), "test_table");
    List<ColumnMetadata> columnList = tblMeta.getColumns();
    assertNotNull(columnList);

    boolean foundServiceType = false;
    boolean foundPartitionKey = false;
    for (ColumnMetadata column : columnList) {
        if (column.getName().equals("service_type")) {
            foundServiceType = true;
            assertEquals(column.getType().getDisplayName(), "varchar(20)");
        }
        if (column.getName().equals("_partition_key")) {
            foundPartitionKey = true;
            assertEquals(column.getType().getDisplayName(), "varchar");
        }
    }
    assertTrue(foundServiceType);
    assertTrue(foundPartitionKey);
}
 

@Override
public void createTable(ConnectorSession session, ConnectorTableMetadata tableMetadata, boolean ignoreExisting) {
    clientSession.createTable(tableMetadata, ignoreExisting);
}
 

private ConnectorTableMetadata getTableMetadataInternal(ConnectorSession session,
                                                        ConnectorTableHandle tableHandle) {
    KuduTableHandle kuduTableHandle = fromConnectorTableHandle(session, tableHandle);
    return getTableMetadata(kuduTableHandle);
}
 

@Override
public ConnectorTableMetadata getTableMetadata(ConnectorSession session, ConnectorTableHandle tableHandle) {
    return getTableMetadataInternal(session, tableHandle);
}
 

@Override
public void createTable(ConnectorSession session, ConnectorTableMetadata tableMetadata, boolean ignoreExisting)
{
    client.createTable(tableMetadata);
}
 

@SuppressWarnings("ValueOfIncrementOrDecrementUsed")
private ConnectorTableMetadata getTableMetadata(SchemaTableName schemaTableName) {
    ImmutableList.Builder<ColumnMetadata> builder = ImmutableList.builder();

    if (EthereumTable.BLOCK.getName().equals(schemaTableName.getTableName())) {
        builder.add(new ColumnMetadata("block_number", BigintType.BIGINT));
        builder.add(new ColumnMetadata("block_hash", VarcharType.createVarcharType(H32_BYTE_HASH_STRING_LENGTH)));
        builder.add(new ColumnMetadata("block_parentHash", VarcharType.createVarcharType(H32_BYTE_HASH_STRING_LENGTH)));
        builder.add(new ColumnMetadata("block_nonce", VarcharType.createVarcharType(H8_BYTE_HASH_STRING_LENGTH)));
        builder.add(new ColumnMetadata("block_sha3Uncles", VarcharType.createVarcharType(H32_BYTE_HASH_STRING_LENGTH)));
        builder.add(new ColumnMetadata("block_logsBloom", VarcharType.createVarcharType(H256_BYTE_HASH_STRING_LENGTH)));
        builder.add(new ColumnMetadata("block_transactionsRoot", VarcharType.createVarcharType(H32_BYTE_HASH_STRING_LENGTH)));
        builder.add(new ColumnMetadata("block_stateRoot", VarcharType.createVarcharType(H32_BYTE_HASH_STRING_LENGTH)));
        builder.add(new ColumnMetadata("block_miner", VarcharType.createVarcharType(H20_BYTE_HASH_STRING_LENGTH)));
        builder.add(new ColumnMetadata("block_difficulty", BigintType.BIGINT));
        builder.add(new ColumnMetadata("block_totalDifficulty", BigintType.BIGINT));
        builder.add(new ColumnMetadata("block_size", IntegerType.INTEGER));
        builder.add(new ColumnMetadata("block_extraData", VarcharType.VARCHAR));
        builder.add(new ColumnMetadata("block_gasLimit", DoubleType.DOUBLE));
        builder.add(new ColumnMetadata("block_gasUsed", DoubleType.DOUBLE));
        builder.add(new ColumnMetadata("block_timestamp", BigintType.BIGINT));
        builder.add(new ColumnMetadata("block_transactions", new ArrayType(VarcharType.createVarcharType(H32_BYTE_HASH_STRING_LENGTH))));
        builder.add(new ColumnMetadata("block_uncles", new ArrayType(VarcharType.createVarcharType(H32_BYTE_HASH_STRING_LENGTH))));
    } else if (EthereumTable.TRANSACTION.getName().equals(schemaTableName.getTableName())) {
        builder.add(new ColumnMetadata("tx_hash", VarcharType.createVarcharType(H32_BYTE_HASH_STRING_LENGTH)));
        builder.add(new ColumnMetadata("tx_nonce", BigintType.BIGINT));
        builder.add(new ColumnMetadata("tx_blockHash", VarcharType.createVarcharType(H32_BYTE_HASH_STRING_LENGTH)));
        builder.add(new ColumnMetadata("tx_blockNumber", BigintType.BIGINT));
        builder.add(new ColumnMetadata("tx_transactionIndex", IntegerType.INTEGER));
        builder.add(new ColumnMetadata("tx_from", VarcharType.createVarcharType(H20_BYTE_HASH_STRING_LENGTH)));
        builder.add(new ColumnMetadata("tx_to", VarcharType.createVarcharType(H20_BYTE_HASH_STRING_LENGTH)));
        builder.add(new ColumnMetadata("tx_value", DoubleType.DOUBLE));
        builder.add(new ColumnMetadata("tx_gas", DoubleType.DOUBLE));
        builder.add(new ColumnMetadata("tx_gasPrice", DoubleType.DOUBLE));
        builder.add(new ColumnMetadata("tx_input", VarcharType.VARCHAR));
    } else if (EthereumTable.ERC20.getName().equals(schemaTableName.getTableName())) {
        builder.add(new ColumnMetadata("erc20_token", VarcharType.createUnboundedVarcharType()));
        builder.add(new ColumnMetadata("erc20_from", VarcharType.createVarcharType(H20_BYTE_HASH_STRING_LENGTH)));
        builder.add(new ColumnMetadata("erc20_to", VarcharType.createVarcharType(H20_BYTE_HASH_STRING_LENGTH)));
        builder.add(new ColumnMetadata("erc20_value", DoubleType.DOUBLE));
        builder.add(new ColumnMetadata("erc20_txHash", VarcharType.createVarcharType(H32_BYTE_HASH_STRING_LENGTH)));
        builder.add(new ColumnMetadata("erc20_blockNumber", BigintType.BIGINT));
    } else {
        throw new IllegalArgumentException("Unknown Table Name " + schemaTableName.getTableName());
    }

    return new ConnectorTableMetadata(schemaTableName, builder.build());
}
 

@Override
public ConnectorTableMetadata getTableMetadata(ConnectorSession session, ConnectorTableHandle tableHandle) {
    return getTableMetadata(convertTableHandle(tableHandle).toSchemaTableName());
}