com.facebook.presto.spi.connector.ConnectorTransactionHandle Java Examples

@Override
public ConnectorSplitSource getSplits(
        ConnectorTransactionHandle transactionHandle,
        ConnectorSession session,
        ConnectorTableLayoutHandle layout,
        SplitSchedulingContext splitSchedulingContext)
{
    KubeTableLayoutHandle layoutHandle = (KubeTableLayoutHandle) layout;
    KubeTableHandle tableHandle = layoutHandle.getTable();

    TupleDomain<KubeColumnHandle> effectivePredicate = layoutHandle.getConstraint()
            .transform(KubeColumnHandle.class::cast);

    List<ConnectorSplit> splits = nodeManager.getAllNodes().stream()
            .map(node -> new KubeSplit(node.getHostAndPort(), tableHandle.getSchemaTableName(), effectivePredicate))
            .collect(Collectors.toList());

    return new FixedSplitSource(splits);
}
 

@Override
public ConnectorSplitSource getSplits(ConnectorTransactionHandle transactionHandle, ConnectorSession session, ConnectorTableLayoutHandle layout, ConnectorSplitManager.SplitSchedulingStrategy splitSchedulingStrategy)
{
    KinesisTableLayoutHandle kinesislayout = handleResolver.convertLayout(layout);
    KinesisTableHandle kinesisTableHandle = kinesislayout.getTable();

    InternalStreamDescription desc = this.getStreamDescription(kinesisTableHandle.getStreamName());

    ImmutableList.Builder<ConnectorSplit> builder = ImmutableList.builder();
    for (Shard shard : desc.getShards()) {
        KinesisSplit split = new KinesisSplit(connectorId,
                kinesisTableHandle.getStreamName(),
                kinesisTableHandle.getMessageDataFormat(),
                shard.getShardId(),
                shard.getSequenceNumberRange().getStartingSequenceNumber(),
                shard.getSequenceNumberRange().getEndingSequenceNumber());
        builder.add(split);
    }

    return new FixedSplitSource(builder.build());
}
 

@Override
public RecordSet getRecordSet(
        ConnectorTransactionHandle transaction,
        ConnectorSession session,
        ConnectorSplit split,
        List<? extends ColumnHandle> columns
) {
    EthereumSplit ethereumSplit = convertSplit(split);

    ImmutableList.Builder<EthereumColumnHandle> handleBuilder = ImmutableList.builder();

    for (ColumnHandle handle : columns) {
        EthereumColumnHandle columnHandle = convertColumnHandle(handle);
        handleBuilder.add(columnHandle);
    }

    return new EthereumRecordSet(web3j, handleBuilder.build(), ethereumSplit);
}
 

@Override
public ConnectorPageSource createPageSource(ConnectorTransactionHandle transactionHandle, ConnectorSession session,
                                            ConnectorSplit split, List<ColumnHandle> columns)
{
    List<ParaflowColumnHandle> hdfsColumns = columns.stream()
            .map(col -> (ParaflowColumnHandle) col)
            .collect(Collectors.toList());
    ParaflowSplit paraflowSplit = checkType(split, ParaflowSplit.class, "hdfs split");
    Path path = new Path(paraflowSplit.getPath());

    Optional<ConnectorPageSource> pageSource = createParaflowPageSource(
            path,
            paraflowSplit.getStart(),
            paraflowSplit.getLen(),
            hdfsColumns);
    if (pageSource.isPresent()) {
        return pageSource.get();
    }
    throw new RuntimeException("Could not find a file reader for split " + paraflowSplit);
}
 

@Parameters({
        "kinesis.awsAccessKey",
        "kinesis.awsSecretKey"
})
@Test
public void testSpinUp(String awsAccessKey, String awsSecretKey)
{
    ConnectorFactory factory = testConnectorExists();
    // Important: this has to be created before we setup the injector in the factory:
    assertNotNull(factory.getHandleResolver());

    Connector c = factory.create("kinesis.test-connector", ImmutableMap.<String, String>builder()
            .put("kinesis.hide-internal-columns", "false")
            .put("kinesis.access-key", TestUtils.noneToBlank(awsAccessKey))
            .put("kinesis.secret-key", TestUtils.noneToBlank(awsSecretKey))
            .build(), new TestingConnectorContext() {});
    assertNotNull(c);

    // Verify that the key objects have been created on the connector
    assertNotNull(c.getRecordSetProvider());
    assertNotNull(c.getSplitManager());
    ConnectorMetadata md = c.getMetadata(KinesisTransactionHandle.INSTANCE);
    assertNotNull(md);

    ConnectorTransactionHandle handle = c.beginTransaction(READ_COMMITTED, true);
    assertTrue(handle != null && handle instanceof KinesisTransactionHandle);
}
 

@Override
public RecordSet getRecordSet(ConnectorTransactionHandle transactionHandle, ConnectorSession session, ConnectorSplit split, List<? extends ColumnHandle> columns)
{
    requireNonNull(split, "split is null");
    KubeSplit kubeSplit = (KubeSplit) split;

    ImmutableList.Builder<KubeColumnHandle> handles = ImmutableList.builder();
    for (ColumnHandle handle : columns) {
        handles.add((KubeColumnHandle) handle);
    }

    return new KubeRecordSet(kubeTables, kubeSplit, handles.build());
}
 

@Override
public RecordSet getRecordSet(ConnectorTransactionHandle transactionHandle, ConnectorSession session,
                              ConnectorSplit split, List<? extends ColumnHandle> columns)
{
    KinesisSplit kinesisSplit = handleResolver.convertSplit(split);

    ImmutableList.Builder<KinesisColumnHandle> handleBuilder = ImmutableList.builder();
    ImmutableMap.Builder<KinesisColumnHandle, KinesisFieldDecoder<?>> messageFieldDecoderBuilder = ImmutableMap.builder();

    KinesisRowDecoder messageDecoder = registry.getRowDecoder(kinesisSplit.getMessageDataFormat());

    for (ColumnHandle handle : columns) {
        KinesisColumnHandle columnHandle = handleResolver.convertColumnHandle(handle);
        handleBuilder.add(columnHandle);

        if (!columnHandle.isInternal()) {
            KinesisFieldDecoder<?> fieldDecoder = registry.getFieldDecoder(kinesisSplit.getMessageDataFormat(),
                    columnHandle.getType().getJavaType(),
                    columnHandle.getDataFormat());

            messageFieldDecoderBuilder.put(columnHandle, fieldDecoder);
        }
    }

    ImmutableList<KinesisColumnHandle> handles = handleBuilder.build();
    ImmutableMap<KinesisColumnHandle, KinesisFieldDecoder<?>> messageFieldDecoders = messageFieldDecoderBuilder.build();

    return new KinesisRecordSet(kinesisSplit, session, clientManager, handles, messageDecoder, messageFieldDecoders, kinesisConnectorConfig);
}
 

@Override
public RecordSet getRecordSet(ConnectorTransactionHandle transactionHandle,
                              ConnectorSession session, ConnectorSplit split, List<? extends ColumnHandle> columns) {
    requireNonNull(split, "split is null");
    requireNonNull(columns, "columns is null");

    KuduSplit kuduSplit = checkType(split, KuduSplit.class, "split is not class KuduSplit");

    return new KuduRecordSet(clientSession, kuduSplit, columns);
}
 

@Override
public ConnectorPageSource createPageSource(ConnectorTransactionHandle transactionHandle,
                                            ConnectorSession session, ConnectorSplit split, List<ColumnHandle> columns) {
    KuduRecordSet recordSet = (KuduRecordSet) recordSetProvider.getRecordSet(transactionHandle, session, split, columns);
    if (columns.contains(KuduColumnHandle.ROW_ID_HANDLE)) {
        return new KuduUpdatablePageSource(recordSet);
    } else {
        return new RecordPageSource(recordSet);
    }
}
 

@Override
public ConnectorPageSink createPageSink(ConnectorTransactionHandle transactionHandle, ConnectorSession session, ConnectorInsertTableHandle insertTableHandle) {
    requireNonNull(insertTableHandle, "insertTableHandle is null");
    checkArgument(insertTableHandle instanceof KuduInsertTableHandle, "insertTableHandle is not an instance of KuduInsertTableHandle");
    KuduInsertTableHandle handle = (KuduInsertTableHandle) insertTableHandle;

    return new KuduPageSink(session, clientSession, handle, false);
}
 

@Override
public ConnectorPageSink createPageSink(ConnectorTransactionHandle transactionHandle, ConnectorSession session, ConnectorOutputTableHandle outputTableHandle) {
    requireNonNull(outputTableHandle, "outputTableHandle is null");
    checkArgument(outputTableHandle instanceof KuduOutputTableHandle, "outputTableHandle is not an instance of KuduOutputTableHandle");
    KuduOutputTableHandle handle = (KuduOutputTableHandle) outputTableHandle;

    return new KuduPageSink(session, clientSession, handle, handle.isGenerateUUID());
}
 

@Override
public ConnectorSplitSource getSplits(ConnectorTransactionHandle transactionHandle,
                                      ConnectorSession session, ConnectorTableLayoutHandle layout,
                                      SplitSchedulingStrategy splitSchedulingStrategy) {
    KuduTableLayoutHandle layoutHandle = (KuduTableLayoutHandle) layout;

    List<KuduSplit> splits = clientSession.buildKuduSplits(layoutHandle);

    return new FixedSplitSource(splits);
}
 

/**
 * Guaranteed to be called at most once per transaction. The returned metadata will only be accessed
 * in a single threaded context.
 *
 * @param transactionHandle transaction handle
 */
@Override
public ConnectorMetadata getMetadata(ConnectorTransactionHandle transactionHandle)
{
    ParaflowMetadata metadata = transactions.get(transactionHandle);
    checkArgument(metadata != null, "no such transaction: %s", transactionHandle);
    return paraflowMetadataFactory.create();
}
 

@Override
public ConnectorTransactionHandle beginTransaction(IsolationLevel isolationLevel, boolean readOnly)
{
    checkConnectorSupports(READ_UNCOMMITTED, isolationLevel);
    ParaflowTransactionHandle transaction = new ParaflowTransactionHandle();
    transactions.putIfAbsent(transaction, paraflowMetadataFactory.create());
    return transaction;
}
 

@Override
public RecordSet getRecordSet(ConnectorTransactionHandle transactionHandle, ConnectorSession session, ConnectorSplit split, List<? extends ColumnHandle> columns)
{
    requireNonNull(split, "split is null");
    HbaseSplit hbaseSplit = (HbaseSplit) split;
    checkArgument(hbaseSplit.getConnectorId().equals(connectorId), "split is not for this connector");

    ImmutableList.Builder<HbaseColumnHandle> handles = ImmutableList.builder();
    for (ColumnHandle handle : columns) {
        handles.add((HbaseColumnHandle) handle);
    }

    return new HbaseRecordSet(hbaseClient, session, hbaseSplit, handles.build());
}
 

@Override
public ConnectorTransactionHandle beginTransaction(IsolationLevel isolationLevel, boolean readOnly)
{
    checkConnectorSupports(READ_UNCOMMITTED, isolationLevel);
    ConnectorTransactionHandle transaction = new ElasticsearchTransactionHandle();
    transactions.put(transaction, new ElasticsearchMetadata(connectorId, client));
    return transaction;
}
 

@Override
public void rollback(ConnectorTransactionHandle transactionHandle)
{
    ElasticsearchMetadata metadata = transactions.remove(transactionHandle);
    checkArgument(metadata != null, "no such transaction: %s", transactionHandle);
    metadata.rollback();
}
 

@Override
public ConnectorSplitSource getSplits(ConnectorTransactionHandle transactionHandle, ConnectorSession session, ConnectorTableLayoutHandle layout, SplitSchedulingStrategy splitSchedulingStrategy)
{
    HbaseTableLayoutHandle layoutHandle = (HbaseTableLayoutHandle) layout;
    HbaseTableHandle tableHandle = layoutHandle.getTable();

    String schemaName = tableHandle.getSchema();
    String tableName = tableHandle.getTable();
    String rowIdName = tableHandle.getRowId();

    // Get non-row ID column constraints
    List<HbaseColumnConstraint> constraints = getColumnConstraints(rowIdName, layoutHandle.getConstraint());

    // Get the row domain column range
    Optional<Domain> rDom = getRangeDomain(rowIdName, layoutHandle.getConstraint());

    // Call out to our client to retrieve all tablet split metadata using the row ID domain and the secondary index
    List<TabletSplitMetadata> tabletSplits = client.getTabletSplits(session, schemaName, tableName, rDom, constraints); //tableHandle.getSerializerInstance()

    // Pack the tablet split metadata into a connector split
    ImmutableList.Builder<ConnectorSplit> cSplits = ImmutableList.builder();
    for (TabletSplitMetadata splitMetadata : tabletSplits) {
        HbaseSplit split = new HbaseSplit(
                connectorId,
                schemaName,
                tableName,
                rowIdName,
                splitMetadata,
                constraints,
                tableHandle.getScanAuthorizations());
        cSplits.add(split);
    }

    return new FixedSplitSource(cSplits.build());
}
 

@Override
    public ConnectorSplitSource getSplits(ConnectorTransactionHandle transactionHandle, ConnectorSession session, ConnectorTableLayoutHandle layout, SplitSchedulingContext splitSchedulingContext )
    {
        SplitSchedulingStrategy splitSchedulingStrategy = splitSchedulingContext.getSplitSchedulingStrategy();
        ElasticsearchTableLayoutHandle layoutHandle = (ElasticsearchTableLayoutHandle) layout;
        ElasticsearchTableHandle tableHandle = layoutHandle.getTable();

//        // Call out to our client to retrieve all tablet split metadata using the row ID domain and the secondary index
        List<ElasticsearchSplit> tabletSplits = client.getTabletSplits(session, tableHandle, layoutHandle, splitSchedulingStrategy); //tableHandle.getSerializerInstance()

        // Pack the tablet split metadata into a connector split
        return new FixedSplitSource(tabletSplits);
    }
 

@Override
public ConnectorPageSource createPageSource(ConnectorTransactionHandle transactionHandle, ConnectorSession session, ConnectorSplit split, List<ColumnHandle> columns)
{
    ElasticsearchSplit elasticsearchSplit = (ElasticsearchSplit) split;

    ImmutableList.Builder<ElasticsearchColumnHandle> handles = ImmutableList.builder();
    for (ColumnHandle handle : requireNonNull(columns, "columns is null")) {
        handles.add((ElasticsearchColumnHandle) handle);
    }

    return new ElasticsearchPageSource(elasticsearchClient, elasticsearchSplit, handles.build());
}
 

@Override
public ConnectorPageSink createPageSink(ConnectorTransactionHandle transactionHandle, ConnectorSession session, ConnectorInsertTableHandle insertTableHandle, PageSinkProperties pageSinkProperties) {
    ElasticsearchTableHandle tableHandle = (ElasticsearchTableHandle) insertTableHandle;
    ElasticsearchTable table = client.getTable(tableHandle.getSchemaTableName());
    List<ElasticsearchColumnHandle> columns = table.getColumns().stream()
            .filter(x -> !x.isHidden()).collect(Collectors.toList());
    return new ElasticsearchPageSink(client, tableHandle.getSchemaTableName(), columns);
}
 

/**
 * lock
 */
@Override
public ConnectorMetadata getMetadata(ConnectorTransactionHandle transactionHandle)
{
    ConnectorMetadata metadata = transactions.get(transactionHandle);
    checkArgument(metadata != null, "no such transaction: %s", transactionHandle);
    return metadata;
}
 

@Override
public Class<? extends ConnectorTransactionHandle> getTransactionHandleClass()
{
    return ElasticsearchTransactionHandle.class;
}
 

@Override
public ConnectorTransactionHandle beginTransaction(IsolationLevel isolationLevel,
                                                   boolean readOnly) {
    checkConnectorSupports(READ_COMMITTED, isolationLevel);
    return KuduTransactionHandle.INSTANCE;
}
 

@Override
public ConnectorMetadata getMetadata(ConnectorTransactionHandle transactionHandle) {
    return metadata;
}
 

@Override
public ConnectorTransactionHandle beginTransaction(IsolationLevel isolationLevel, boolean readOnly) {
	// TODO Auto-generated method stub
	return null;
}
 

@Override
public ConnectorMetadata getMetadata(ConnectorTransactionHandle transactionHandle) {
	// TODO Auto-generated method stub
	return null;
}
 

@Override
public Class<? extends ConnectorTransactionHandle> getTransactionHandleClass()
{
    return KubeTransactionHandle.class;
}
 

@Override
public Class<? extends ConnectorTransactionHandle> getTransactionHandleClass()
{
    return KinesisTransactionHandle.class;
}
 

@Override
public ConnectorMetadata getMetadata(ConnectorTransactionHandle transactionHandle)
{
    return metadata;
}