com.facebook.presto.spi.SchemaTableName Java Examples

@Parameters({
    "kinesis.awsAccessKey",
    "kinesis.awsSecretKey"
})
@BeforeMethod
public void spinUp(String accessKey, String secretKey)
        throws Exception
{
    streamName = "test_" + UUID.randomUUID().toString().replaceAll("-", "_");
    embeddedKinesisStream.createStream(2, streamName);
    this.queryRunner = new StandaloneQueryRunner(SESSION);
    TestUtils.installKinesisPlugin(queryRunner,
            ImmutableMap.<SchemaTableName, KinesisStreamDescription>builder().
            put(TestUtils.createEmptyStreamDescription(streamName, new SchemaTableName("default", streamName))).build(),
            TestUtils.noneToBlank(accessKey), TestUtils.noneToBlank(secretKey));
}
 

@Override
public List<SchemaTableName> listTables(ConnectorSession session, String schemaNameOrNull)
{
    Set<String> schemaNames;
    if (schemaNameOrNull != null) {
        schemaNames = ImmutableSet.of(schemaNameOrNull);
    }
    else {
        schemaNames = client.getSchemaNames();
    }
    ImmutableList.Builder<SchemaTableName> builder = ImmutableList.builder();
    for (String schemaName : schemaNames) {
        for (String tableName : client.getTableNames(schemaName)) {
            builder.add(new SchemaTableName(schemaName, tableName));
        }
    }
    return builder.build();
}
 

@Override
public List<SchemaTableName> listTables(ConnectorSession session, String schemaNameOrNull)
{
    Set<String> schemaNames;
    if (schemaNameOrNull != null) {
        schemaNames = ImmutableSet.of(schemaNameOrNull);
    }
    else {
        schemaNames = client.getSchemaNames();
    }
    ImmutableList.Builder<SchemaTableName> builder = ImmutableList.builder();
    for (String schemaName : schemaNames) {
        for (String tableName : client.getTableNames(schemaName)) {
            builder.add(new SchemaTableName(schemaName, tableName));
        }
    }
    return builder.build();
}
 

@Inject
KinesisMetadata(@Named("connectorId") String connectorId,
        KinesisConnectorConfig kinesisConnectorConfig,
        KinesisHandleResolver handleResolver,
        Supplier<Map<SchemaTableName, KinesisStreamDescription>> kinesisTableDescriptionSupplier,
        Set<KinesisInternalFieldDescription> internalFieldDescriptions)
{
    this.connectorId = checkNotNull(connectorId, "connectorId is null");
    this.kinesisConnectorConfig = checkNotNull(kinesisConnectorConfig, "kinesisConfig is null");
    this.handleResolver = checkNotNull(handleResolver, "handleResolver is null");

    log.debug("Loading kinesis table definitions from %s", kinesisConnectorConfig.getTableDescriptionDir());

    this.kinesisTableDescriptionSupplier = kinesisTableDescriptionSupplier;
    this.internalFieldDescriptions = checkNotNull(internalFieldDescriptions, "internalFieldDescriptions is null");
}
 

private List<SchemaTableName> listTables(ConnectorSession session, SchemaTablePrefix prefix)
{
    // List all tables if schema or table is null
    if (prefix.getSchemaName() == null || prefix.getTableName() == null) {
        return listTables(session, prefix.getSchemaName());
    }

    // Make sure requested table exists, returning the single table of it does
    SchemaTableName table = new SchemaTableName(prefix.getSchemaName(), prefix.getTableName());
    if (getTableHandle(session, table) != null) {
        return ImmutableList.of(table);
    }

    // Else, return empty list
    return ImmutableList.of();
}
 

@Override
public Map<SchemaTableName, List<ColumnMetadata>> listTableColumns(ConnectorSession session, SchemaTablePrefix prefix)
{
    checkNotNull(prefix, "prefix is null");
    log.debug("Called listTableColumns on %s.%s", prefix.getSchemaName(), prefix.getTableName());

    ImmutableMap.Builder<SchemaTableName, List<ColumnMetadata>> columns = ImmutableMap.builder();

    // NOTE: prefix.getTableName or prefix.getSchemaName can be null
    List<SchemaTableName> tableNames;
    if (prefix.getSchemaName() != null && prefix.getTableName() != null) {
        tableNames = ImmutableList.of(new SchemaTableName(prefix.getSchemaName(), prefix.getTableName()));
    }
    else {
        tableNames = listTables(session, (String) null);
    }

    for (SchemaTableName tableName : tableNames) {
        ConnectorTableMetadata tableMetadata = getTableMetadata(tableName);
        if (tableMetadata != null) {
            columns.put(tableName, tableMetadata.getColumns());
        }
    }
    return columns.build();
}
 

public Map<SchemaTableName, KinesisStreamDescription> getTablesFromDirectory()
{
    ImmutableMap.Builder<SchemaTableName, KinesisStreamDescription> builder = ImmutableMap.builder();
    try {
        for (Path file : listFiles(Paths.get(kinesisConnectorConfig.getTableDescriptionDir()))) {
            if (Files.isRegularFile(file) && file.getFileName().toString().endsWith("json")) {
                KinesisStreamDescription table = streamDescriptionCodec.fromJson(Files.readAllBytes(file));
                String schemaName = Objects.firstNonNull(table.getSchemaName(), kinesisConnectorConfig.getDefaultSchema());
                log.debug("Kinesis table %s %s %s", schemaName, table.getTableName(), table);
                builder.put(new SchemaTableName(schemaName, table.getTableName()), table);
            }
        }

        Map<SchemaTableName, KinesisStreamDescription> tableDefinitions = builder.build();
        log.debug("Loaded table definitions: %s", tableDefinitions.keySet());

        return tableDefinitions;
    }
    catch (IOException e) {
        log.warn(e, "Error: ");
        throw Throwables.propagate(e);
    }
}
 

@Override
public Map<SchemaTableName, List<ColumnMetadata>> listTableColumns(ConnectorSession session,
                                                                   SchemaTablePrefix prefix) {
    requireNonNull(prefix, "SchemaTablePrefix is null");

    List<SchemaTableName> tables;
    if (prefix.getSchemaName() == null) {
        tables = listTables(session, Optional.empty());
    } else if (prefix.getTableName() == null) {
        tables = listTables(session, prefix.getSchemaName());
    } else {
        tables = ImmutableList.of(new SchemaTableName(prefix.getSchemaName(), prefix.getTableName()));
    }

    ImmutableMap.Builder<SchemaTableName, List<ColumnMetadata>> columns = ImmutableMap.builder();
    for (SchemaTableName tableName : tables) {
        KuduTableHandle tableHandle = getTableHandle(session, tableName);
        ConnectorTableMetadata tableMetadata = getTableMetadata(tableHandle);
        columns.put(tableName, tableMetadata.getColumns());
    }
    return columns.build();
}
 

@Test
public void testTableDefinition()
{
    // Get the supplier from the injector
    KinesisTableDescriptionSupplier supplier = TestUtils.getTableDescSupplier(injector);
    assertNotNull(supplier);

    // Read table definition and verify
    Map<SchemaTableName, KinesisStreamDescription> readMap = supplier.get();
    assertTrue(!readMap.isEmpty());

    SchemaTableName tblName = new SchemaTableName("prod", "test_table");
    KinesisStreamDescription desc = readMap.get(tblName);

    assertNotNull(desc);
    assertEquals(desc.getSchemaName(), "prod");
    assertEquals(desc.getTableName(), "test_table");
    assertEquals(desc.getStreamName(), "test_kinesis_stream");
    assertNotNull(desc.getMessage());

    // Obtain the message part and verify we can read its fields
    KinesisStreamFieldGroup grp = desc.getMessage();
    assertEquals(grp.getDataFormat(), "json");
    List<KinesisStreamFieldDescription> fieldList = grp.getFields();
    assertEquals(fieldList.size(), 4); // (4 fields in test_table.json)
}
 

@Override
public List<SchemaTableName> listTables(ConnectorSession session, String schemaNameOrNull)
{
    List<String> schemaNames;
    if (schemaNameOrNull != null) {
        schemaNames = ImmutableList.of(schemaNameOrNull);
    }
    else {
        schemaNames = paraflowMetaDataReader.getAllDatabases();
    }

    ImmutableList.Builder<SchemaTableName> builder = ImmutableList.builder();
    for (String schemaName : schemaNames) {
        for (String tableName : paraflowMetaDataReader.getTableNames(schemaName)) {
            builder.add(new SchemaTableName(schemaName, tableName));
        }
    }
    return builder.build();
}
 

public List<SchemaTableName> listTables(String dbPrefix)
{
    List<SchemaTableName> tables = new ArrayList<>();

    // if dbPrefix not mean to match all
    String tblName;
    String dbName;
    if (dbPrefix != null) {
        MetaProto.StringListType stringListType = metaClient.listTables(dbPrefix);
        if (stringListType.getStrCount() == 0) {
            return tables;
        }
        for (int i = 0; i < stringListType.getStrCount(); i++) {
            tblName = stringListType.getStr(0);
            dbName = dbPrefix;
            tables.add(new SchemaTableName(dbName, tblName));
        }
    }
    return tables;
}
 

@BeforeMethod
public void spinUp() throws Exception {
    ImmutableMap<SchemaTableName, KinesisStreamDescription> streamMap =
            ImmutableMap.<SchemaTableName, KinesisStreamDescription>builder().
                    put(TestUtils.createEmptyStreamDescription(dummyStreamName, new SchemaTableName("default", dummyStreamName))).
                    put(TestUtils.createSimpleJsonStreamDescription(jsonStreamName, new SchemaTableName("default", jsonStreamName))).
                    build();
    this.queryRunner = new StandaloneQueryRunner(SESSION);
    KinesisPlugin plugin = TestUtils.installKinesisPlugin(queryRunner, streamMap);
    clientManager = TestUtils.getTestClientManager(plugin.getInjector());
    mockClient = (MockKinesisClient) clientManager.getClient();

    mockClient.createStream(dummyStreamName, 2);
    mockClient.createStream(jsonStreamName, 2);

    log.info("Completed spinUp steps.  *** READY FOR QUERIES ***");
}
 

public void dropTable(HbaseTable table)
{
    SchemaTableName tableName = new SchemaTableName(table.getSchema(), table.getTable());

    // Remove the table metadata from Presto
    if (metaManager.getTable(tableName) != null) {
        metaManager.deleteTableMetadata(tableName);
    }

    if (!table.isExternal()) {
        // delete the table
        String fullTableName = table.getFullTableName();
        if (tableManager.exists(fullTableName)) {
            tableManager.deleteHbaseTable(fullTableName);
        }
    }
}
 

@Override
public void dropSchema(String schemaName) {
    if (DEFAULT_SCHEMA.equals(schemaName)) {
        throw new PrestoException(GENERIC_USER_ERROR, "Deleting default schema not allowed.");
    }
    else {
        try {
            for (SchemaTableName table : listTables(schemaName)) {
                dropTable(table);
            }
            KuduTable schemasTable = getSchemasTable();
            KuduSession session = client.newSession();
            try {
                Delete delete = schemasTable.newDelete();
                fillSchemaRow(delete.getRow(), schemaName);
                session.apply(delete);
            }
            finally {
                session.close();
            }
        }
        catch (KuduException e) {
            throw new PrestoException(GENERIC_INTERNAL_ERROR, e);
        }
    }
}
 

@Override
public ConnectorTableMetadata getTableMetadata(ConnectorSession session, ConnectorTableHandle table)
{
    HbaseTableHandle handle = (HbaseTableHandle) table;
    checkArgument(handle.getConnectorId().equals(connectorId), "table is not for this connector");
    SchemaTableName tableName = new SchemaTableName(handle.getSchema(), handle.getTable());
    ConnectorTableMetadata metadata = getTableMetadata(tableName);
    if (metadata == null) {
        throw new TableNotFoundException(tableName);
    }
    return metadata;
}
 

public void deleteTableMetadata(SchemaTableName tableName)
{
    try {
        curator.delete().deletingChildrenIfNeeded().forPath(getTablePath(tableName));
    }
    catch (Exception e) {
        throw new PrestoException(ZOOKEEPER_ERROR, "ZK error when deleting table metadata", e);
    }
}
 

@Override
public Map<SchemaTableName, KinesisStreamDescription> get()
{
    if (this.s3TableConfigClient.isUsingS3()) {
        return this.s3TableConfigClient.getTablesFromS3();
    }
    else {
        return getTablesFromDirectory();
    }
}
 

private String toRawName(SchemaTableName schemaTableName) {
    String rawName;
    if (schemaTableName.getSchemaName().equals(DEFAULT_SCHEMA)) {
        rawName = tenantPrefix + schemaTableName.getTableName();
    } else {
        rawName = tenantPrefix + schemaTableName.getSchemaName() + "." + schemaTableName.getTableName();
    }
    return rawName;
}
 

public KuduExtendedTableHandle(String connectorId, SchemaTableName schemaTableName,
                               List<String> columnNames, List<Type> columnTypes,
                               KuduTable table) {
    super(connectorId, schemaTableName, table);

    requireNonNull(columnNames, "columnNames is null");
    requireNonNull(columnTypes, "columnTypes is null");
    checkArgument(columnNames.size() == columnTypes.size(), "columnNames and columnTypes sizes don't match");
    this.columnNames = ImmutableList.copyOf(columnNames);
    this.columnTypes = ImmutableList.copyOf(columnTypes);
}
 

@Override
public Map<SchemaTableName, List<ColumnMetadata>> listTableColumns(ConnectorSession session, SchemaTablePrefix prefix)
{
    Map<SchemaTableName, List<ColumnMetadata>> tableColumns = new HashMap<>();
    List<SchemaTableName> tableNames = paraflowMetaDataReader.listTables(prefix);
    for (SchemaTableName table : tableNames) {
        List<ColumnMetadata> columnMetadatas = paraflowMetaDataReader.getTableColMetadata(connectorId, table.getSchemaName(),
                table.getTableName()).orElse(new ArrayList<>());
        tableColumns.putIfAbsent(table, columnMetadatas);
    }
    return tableColumns;
}
 

@Override
public KuduTable openTable(SchemaTableName schemaTableName) {
    String rawName = toRawName(schemaTableName);
    try {
        KuduTable table = client.openTable(rawName);
        return table;
    } catch (Exception e) {
        log.debug("Error on doOpenTable: " + e, e);
        if (!listSchemaNames().contains(schemaTableName.getSchemaName())) {
            throw new SchemaNotFoundException(schemaTableName.getSchemaName());
        } else {
            throw new TableNotFoundException(schemaTableName);
        }
    }
}
 

@Override
public List<SchemaTableName> listTables(ConnectorSession session, String schemaNameOrNull)
{
    return ImmutableList.of(new SchemaTableName(DEFAULT_SCHEMA, EthereumTable.BLOCK.getName()),
            new SchemaTableName(DEFAULT_SCHEMA, EthereumTable.TRANSACTION.getName()),
            new SchemaTableName(DEFAULT_SCHEMA, EthereumTable.ERC20.getName()));
}
 

@Override
public KuduTableHandle getTableHandle(ConnectorSession session, SchemaTableName schemaTableName) {
    try {
        KuduTable table = clientSession.openTable(schemaTableName);
        return new KuduTableHandle(connectorId, schemaTableName, table);
    } catch (Exception e) {
        return null;
    }
}
 

@Override
public List<ConnectorTableLayoutResult> getTableLayouts(ConnectorSession session, ConnectorTableHandle table, Constraint<ColumnHandle> constraint, Optional<Set<ColumnHandle>> desiredColumns)
{
    ParaflowTableHandle paraflowTable = checkType(table, ParaflowTableHandle.class, "table");
    SchemaTableName tableName = paraflowTable.getSchemaTableName();
    ParaflowTableLayoutHandle tableLayout = paraflowMetaDataReader.getTableLayout(connectorId, tableName.getSchemaName(), tableName.getTableName()).orElse(null);
    if (tableLayout == null) {
        return ImmutableList.of();
    }
    tableLayout.setPredicates(constraint.getSummary() != null ? Optional.of(constraint.getSummary()) : Optional.empty());
    ConnectorTableLayout layout = getTableLayout(session, tableLayout);

    return ImmutableList.of(new ConnectorTableLayoutResult(layout, constraint.getSummary()));
}
 

/**
 * Install the plug in into the given query runner, using normal setup but with the given table descriptions.
 *
 * Note that this uses the actual client and will incur charges from AWS when run.  Mainly for full
 * integration tests.
 *
 * @param queryRunner
 * @param streamDescriptions
 * @param accessKey
 * @param secretKey
 */
public static void installKinesisPlugin(QueryRunner queryRunner, Map<SchemaTableName, KinesisStreamDescription> streamDescriptions, String accessKey, String secretKey)
{
    KinesisPlugin kinesisPlugin = createPluginInstance();
    // Note: function literal with provided descriptions instead of KinesisTableDescriptionSupplier:
    kinesisPlugin.setTableDescriptionSupplier(() -> streamDescriptions);
    queryRunner.installPlugin(kinesisPlugin);

    Map<String, String> kinesisConfig = ImmutableMap.of(
                "kinesis.default-schema", "default",
                "kinesis.access-key", accessKey,
                "kinesis.secret-key", secretKey);
    queryRunner.createCatalog("kinesis", "kinesis", kinesisConfig);
}
 

private boolean isHbaseView(SchemaTableName tableName)
{
    try {
        String path = getTablePath(tableName);
        return curator.checkExists().forPath(path) != null && isHbaseView(curator.getData().forPath(path));
    }
    catch (Exception e) {
        // Capture race condition between checkExists and getData
        if (e instanceof KeeperException && ((KeeperException) e).code() == NONODE) {
            return false;
        }

        throw new PrestoException(ZOOKEEPER_ERROR, "Error checking if path is an HbaseView object", e);
    }
}
 

private boolean isHbaseTable(SchemaTableName tableName)
{
    try {
        String path = getTablePath(tableName);
        return curator.checkExists().forPath(path) != null && isHbaseTable(curator.getData().forPath(path));
    }
    catch (Exception e) {
        // Capture race condition between checkExists and getData
        if (e instanceof KeeperException && ((KeeperException) e).code() == NONODE) {
            return false;
        }

        throw new PrestoException(ZOOKEEPER_ERROR, "Error checking if path %s is an HbaseTable object", e);
    }
}
 

@Override
public void dropColumn(SchemaTableName schemaTableName, String name) {
    try {
        String rawName = toRawName(schemaTableName);
        AlterTableOptions alterOptions = new AlterTableOptions();
        alterOptions.dropColumn(name);
        client.alterTable(rawName, alterOptions);
    } catch (KuduException e) {
        throw new PrestoException(GENERIC_INTERNAL_ERROR, e);
    }
}
 

public static Map.Entry<SchemaTableName, KinesisStreamDescription> createSimpleJsonStreamDescription(String streamName, SchemaTableName schemaTableName)
{
    // Format: {"id" : 1324, "name" : "some string"}
    ArrayList<KinesisStreamFieldDescription> fieldList = new ArrayList<KinesisStreamFieldDescription>();
    fieldList.add(new KinesisStreamFieldDescription("id", BigintType.BIGINT, "id", "comment", null, null, false));
    fieldList.add(new KinesisStreamFieldDescription("name", VarcharType.VARCHAR, "name", "comment", null, null, false));
    KinesisStreamFieldGroup grp = new KinesisStreamFieldGroup("json", fieldList);

    KinesisStreamDescription desc = new KinesisStreamDescription(schemaTableName.getTableName(), schemaTableName.getSchemaName(), streamName, grp);
    return new AbstractMap.SimpleImmutableEntry<>(schemaTableName, desc);
}
 

@Override
public void dropTable(SchemaTableName schemaTableName) {
    try {
        String rawName = toRawName(schemaTableName);
        client.deleteTable(rawName);
    } catch (KuduException e) {
        throw new PrestoException(GENERIC_INTERNAL_ERROR, e);
    }
}