org.apache.kudu.client.CreateTableOptions Java Examples

public KuduTable createTable(ConnectorTableMetadata tableMetadata, boolean ignoreExisting)
{
    try {
        String rawName = schemaEmulation.toRawName(tableMetadata.getTable());
        if (ignoreExisting) {
            if (client.tableExists(rawName)) {
                return null;
            }
        }

        if (!schemaEmulation.existsSchema(client, tableMetadata.getTable().getSchemaName())) {
            throw new SchemaNotFoundException(tableMetadata.getTable().getSchemaName());
        }

        List<ColumnMetadata> columns = tableMetadata.getColumns();
        Map<String, Object> properties = tableMetadata.getProperties();

        Schema schema = buildSchema(columns, properties);
        CreateTableOptions options = buildCreateTableOptions(schema, properties);
        return client.createTable(rawName, schema, options);
    }
    catch (KuduException e) {
        throw new PrestoException(GENERIC_INTERNAL_ERROR, e);
    }
}
 

private void createAndFillSchemasTable(KuduClient client)
        throws KuduException
{
    List<String> existingSchemaNames = listSchemaNamesFromTablets(client);
    ColumnSchema schemaColumnSchema = new ColumnSchema.ColumnSchemaBuilder("schema", Type.STRING)
            .key(true).build();
    Schema schema = new Schema(ImmutableList.of(schemaColumnSchema));
    CreateTableOptions options = new CreateTableOptions();
    options.addHashPartitions(ImmutableList.of(schemaColumnSchema.getName()), 2);
    KuduTable schemasTable = client.createTable(rawSchemasTableName, schema, options);
    KuduSession session = client.newSession();
    try {
        session.setFlushMode(SessionConfiguration.FlushMode.AUTO_FLUSH_BACKGROUND);
        for (String schemaName : existingSchemaNames) {
            Insert insert = schemasTable.newInsert();
            insert.getRow().addString(0, schemaName);
            session.apply(insert);
        }
    }
    finally {
        session.close();
    }
}
 

@Path("/createTable")
@PUT
public Response createTable() {
    LOG.info("Calling createTable");

    final List<ColumnSchema> columns = new ArrayList<>(2);
    columns.add(new ColumnSchema.ColumnSchemaBuilder("id", Type.STRING).key(true).build());
    columns.add(new ColumnSchema.ColumnSchemaBuilder("name", Type.STRING).build());

    CreateTableOptions cto = new CreateTableOptions().setRangePartitionColumns(Arrays.asList("id")).setNumReplicas(1);

    final Map<String, Object> headers = new HashMap<>();
    headers.put(KuduConstants.CAMEL_KUDU_SCHEMA, new Schema(columns));
    headers.put(KuduConstants.CAMEL_KUDU_TABLE_OPTIONS, cto);

    producerTemplate.requestBodyAndHeaders("direct:create_table", null, headers);

    return Response.ok().build();
}
 

public CreateTableOptions getCreateTableOptions() {
    CreateTableOptions options = new CreateTableOptions();
    if(replicas!=null){
        options.setNumReplicas(replicas);
    }
    if(hasColummns()) {
        List<String> rangeKeys = new ArrayList<>();
        List<String> hashKeys = new ArrayList<>();
        for(KuduColumnInfo column : columns){
            if(column.isRangeKey()){
                rangeKeys.add(column.name());
            }
            if(column.isHashKey()){
                hashKeys.add(column.name());
            }
        }
        options.setRangePartitionColumns(rangeKeys);
        options.addHashPartitions(hashKeys, replicas*2);
    }

    return options;
}
 

public CreateTableOptions getCreateTableOptions() {
    CreateTableOptions options = new CreateTableOptions();
    if(replicas!=null){
        options.setNumReplicas(replicas);
    }
    if(hasColummns()) {
        List<String> rangeKeys = new ArrayList<>();
        List<String> hashKeys = new ArrayList<>();
        for(KuduColumnInfo column : columns){
            if(column.isRangeKey()){
                rangeKeys.add(column.name());
            }
            if(column.isHashKey()){
                hashKeys.add(column.name());
            }
        }
        options.setRangePartitionColumns(rangeKeys);
        options.addHashPartitions(hashKeys, replicas*2);
    }

    return options;
}
 

private void beforeProducer(Exchange exchange) {
    final Message in = exchange.getIn();
    final KuduTable model = exchange.getIn().getBody(KuduTable.class);

    if (model != null && ObjectHelper.isNotEmpty(model.getSchema())) {
        schema = model.getSchema();
    }

    KuduTable.ColumnSchema[] columnSchema = schema.getColumns();
    List<ColumnSchema> columns = new ArrayList<>(columnSchema.length);
    List<String> rangeKeys = new ArrayList<>();
    for (int i = 0; i < columnSchema.length; i++) {
        if (columnSchema[i].isKey()) {
            rangeKeys.add(columnSchema[i].getName());
        }

        columns.add(
                new ColumnSchema.ColumnSchemaBuilder(columnSchema[i].getName(), convertType(columnSchema[i].getType()))
                        .key(columnSchema[i].isKey())
                        .build()
        );
    }

    in.setHeader("Schema", new Schema(columns));
    in.setHeader("TableOptions", new CreateTableOptions().setRangePartitionColumns(rangeKeys));
}
 

protected void createTestTable(final String tableName, final String connection) throws KuduException {
    try (KuduClient client = new KuduClient.KuduClientBuilder(connection).build()) {

        final List<ColumnSchema> columns = new ArrayList<>(5);
        final List<String> columnNames = Arrays.asList("id", "title", "name", "lastname", "address");

        for (int i = 0; i < columnNames.size(); i++) {
            final Type type = i == 0 ? Type.INT32 : Type.STRING;
            columns.add(
                new ColumnSchema.ColumnSchemaBuilder(columnNames.get(i), type)
                    .key(i == 0)
                    .build());
        }

        final List<String> rangeKeys = new ArrayList<>();
        rangeKeys.add("id");

        client.createTable(tableName,
            new Schema(columns),
            new CreateTableOptions().setRangePartitionColumns(rangeKeys));
    }
}
 

@Override
public MetadataWriter createMetadataWriter(final MetadataType metadataType) {
  synchronized (CREATE_TABLE_MUTEX) {
    try {
      if (!metadataExists(metadataType)) {
        final List<ColumnSchema> columns = new ArrayList<>();
        for (final KuduMetadataField f : KuduMetadataField.values()) {
          f.addColumn(columns);
        }
        client.createTable(
            getKuduQualifiedName(getMetadataTableName(metadataType)),
            new Schema(columns),
            new CreateTableOptions().addHashPartitions(
                Collections.singletonList(KuduMetadataField.GW_PRIMARY_ID_KEY.getFieldName()),
                KuduUtils.KUDU_DEFAULT_BUCKETS).setNumReplicas(KuduUtils.KUDU_DEFAULT_REPLICAS));
      }
    } catch (final IOException e) {
      LOGGER.error(
          "Unable to create metadata table '{}'",
          getKuduQualifiedName(getMetadataTableName(metadataType)),
          e);
    }
  }
  return new KuduMetadataWriter(this, metadataType);
}
 

@Test
public void testBeforeProducerFromOptions() throws Exception {
    Map<String, Object> options = new HashMap<>();
    options.put("columns", "Integer,id;String,title;String,name;String,lastname");

    customizer.customize(getComponent(), options);

    Exchange inbound = new DefaultExchange(createCamelContext());
    getComponent().getBeforeProducer().process(inbound);

    Schema schema = (Schema) inbound.getIn().getHeader("Schema");
    CreateTableOptions builder = (CreateTableOptions) inbound.getIn().getHeader("TableOptions");

    Assert.assertNotNull(schema);
    Assert.assertNotNull(builder);

    Assert.assertEquals("Table schema has all elements", 4, schema.getColumnCount());
    Assert.assertEquals("Name of the first column matches", "id", schema.getColumn("id").getName());
    Assert.assertEquals("Type of the first column matches", "int32", schema.getColumn("id").getType().getName());

    Assert.assertEquals("Name of the first column matches", "name", schema.getColumn("name").getName());
    Assert.assertEquals("Type of the first column matches", "string", schema.getColumn("name").getType().getName());
}
 

public static KuduTableInfo booksTableInfo(String tableName, boolean createIfNotExist) {

        KuduTableInfo tableInfo = KuduTableInfo.forTable(tableName);

        if (createIfNotExist) {
            ColumnSchemasFactory schemasFactory = () -> {
                List<ColumnSchema> schemas = new ArrayList<>();
                schemas.add(new ColumnSchema.ColumnSchemaBuilder("id", Type.INT32).key(true).build());
                schemas.add(new ColumnSchema.ColumnSchemaBuilder("title", Type.STRING).build());
                schemas.add(new ColumnSchema.ColumnSchemaBuilder("author", Type.STRING).build());
                schemas.add(new ColumnSchema.ColumnSchemaBuilder("price", Type.DOUBLE).nullable(true).build());
                schemas.add(new ColumnSchema.ColumnSchemaBuilder("quantity", Type.INT32).nullable(true).build());
                return schemas;
            };

            tableInfo.createTableIfNotExists(
                    schemasFactory,
                    () -> new CreateTableOptions()
                            .setNumReplicas(1)
                            .addHashPartitions(Lists.newArrayList("id"), 2));
        }

        return tableInfo;
    }
 

private void createAndFillSchemasTable() throws KuduException {
    List<String> existingSchemaNames = listSchemaNamesFromTablets();
    ColumnSchema tenantColumnSchema = new ColumnSchema.ColumnSchemaBuilder("tenant", Type.STRING)
            .key(true).build();
    ColumnSchema schemaColumnSchema = new ColumnSchema.ColumnSchemaBuilder("schema", Type.STRING)
            .key(true).build();
    Schema schema = new Schema(ImmutableList.of(tenantColumnSchema, schemaColumnSchema));
    CreateTableOptions options = new CreateTableOptions();
    options.setNumReplicas(1); // TODO config
    options.addHashPartitions(ImmutableList.of(tenantColumnSchema.getName()), 2);
    KuduTable schemasTable = client.createTable(rawSchemasTableName, schema, options);
    KuduSession session = client.newSession();
    session.setFlushMode(SessionConfiguration.FlushMode.AUTO_FLUSH_BACKGROUND);
    try {
        for (String schemaName : existingSchemaNames) {
            Insert insert = schemasTable.newInsert();
            fillSchemaRow(insert.getRow(), schemaName);
            session.apply(insert);
        }
    } finally {
        session.close();
    }
}
 

@Override
public KuduTable createTable(ConnectorTableMetadata tableMetadata, boolean ignoreExisting) {
    try {
        SchemaTableName schemeTableName= tableMetadata.getTable();
        String rawName = toRawName(schemeTableName);
        if (ignoreExisting) {
            if (client.tableExists(rawName)) {
                return null;
            }
        }
        if(!schemaExists(schemeTableName.getSchemaName())){
            throw new SchemaNotFoundException(schemeTableName.getSchemaName());
        }
        List<ColumnMetadata> columns = tableMetadata.getColumns();
        Map<String, Object> properties = tableMetadata.getProperties();

        Schema schema = buildSchema(columns, properties);
        CreateTableOptions options = buildCreateTableOptions(schema, properties);
        return client.createTable(rawName, schema, options);
    } catch (KuduException e) {
        throw new PrestoException(GENERIC_INTERNAL_ERROR, e);
    }
}
 

private static Deferred<KuduTable> openOrCreateTable(final AsyncKuduClient client,
                                                     final String table,
                                                     final Schema schema,
                                                     final CreateTableOptions options) throws Exception {
  class CreateTableErrback implements Callback<Deferred<KuduTable>, Exception> {
    @Override
    public Deferred<KuduTable> call(Exception e) throws Exception {
      // TODO(danburkert): we should only do this if the error is "not found"
      LOG.debug("Creating table {}", table);
      return client.createTable(table, schema, options);
    }
    @Override
    public String toString() {
      return MoreObjects.toStringHelper(this).add("table", table).toString();
    }
  }

  return client.openTable(table).addErrback(new CreateTableErrback());
}
 

/**
 * The {@code tagsets} table is range partitioned on the {@code id} column.
 * Because the table is essentially a linear probe hash table, it must be able
 * to be scanned in PK order, so hash partitioning is not possible. Since the
 * tagset IDs are effectively random, setting split points at even intervals
 * over the ID range gives good protection against hotspotting.
 * @param options the create options
 * @param numTabletServers the number of tablet servers
 * @return the tagset table create options
 */
static CreateTableOptions tagsetsCreateTableOptions(CreateOptions options,
                                                    int numTabletServers) {
  CreateTableOptions create = new CreateTableOptions();
  create.setNumReplicas(options.getNumReplicas());

  create.setRangePartitionColumns(ImmutableList.of("id"));

  int numTablets = options.getNumTagsetsTablets(numTabletServers);
  long interval = (1L << 32) / numTablets;
  for (int i = 1; i < numTablets; i++) {
    PartialRow split = TAGSETS_SCHEMA.newPartialRow();
    split.addInt(TAGSETS_ID_INDEX, (int) (Integer.MIN_VALUE + i * interval));
    create.addSplitRow(split);
  }
  return create;
}
 

/**
 * The {@code metrics} table is hash partitioned on the {@code metric} and
 * {@code tagset_id} columns, and range partitioned on the {@code time}
 * column. The hash partitioning allows writes and scans at the current time
 * to be evenly distributed over the cluster. Range partitioning on time
 * allows whole tablets to be pruned based on the time constraint, and allows
 * old data to be dropped if desired.
 * @param options the create options
 * @param numTabletServers the number of tablet servers
 * @return the tags table create options
 */
static CreateTableOptions metricsCreateTableOptions(CreateOptions options,
                                                    int numTabletServers) {
  CreateTableOptions create = new CreateTableOptions();
  create.setNumReplicas(options.getNumReplicas());
  create.addHashPartitions(ImmutableList.of("metric", "tagset_id"),
                           options.getNumMetricsHashBuckets(numTabletServers));
  create.setRangePartitionColumns(ImmutableList.of("time"));
  for (Long time : options.getMetricsSplits()) {
    PartialRow split = METRICS_SCHEMA.newPartialRow();
    split.addLong(METRICS_TIME_INDEX, time);
    create.addSplitRow(split);
  }
  return create;
}
 

/**
 * @return CreateTableOptions if {@link #createTableIfNotExists} was specified.
 */
public CreateTableOptions getCreateTableOptions() {
    if (!getCreateTableIfNotExists()) {
        throw new RuntimeException("Cannot access CreateTableOptions for KuduTableInfo. Use createTableIfNotExists to specify.");
    }
    return createTableOptionsFactory.getCreateTableOptions();
}
 

/**
 * The {@code tags} table is hash partitioned on the {@code key} and
 * {@code value} columns, which allows a scan of a ({@code key}, {@code value})
 * pair to be fully satisfied by a single tablet, and gives good protection
 * against hotspotting.
 * @param options the create options
 * @param numTabletServers the number of tablet servers
 * @return the tags table create options
 */
static CreateTableOptions tagsCreateTableOptions(CreateOptions options,
                                                 int numTabletServers) {
  CreateTableOptions create = new CreateTableOptions();
  create.setNumReplicas(options.getNumReplicas());
  create.addHashPartitions(ImmutableList.of("key", "value"),
                           options.getNumTagsTablets(numTabletServers));
  create.setRangePartitionColumns(ImmutableList.<String>of());
  return create;
}
 

public static void createTestTable(String tableName, KuduClient client) throws Exception
{
  List<String> rangeKeys = new ArrayList<>();
  rangeKeys.add("introwkey");
  List<String> hashPartitions = new ArrayList<>();
  hashPartitions.add("stringrowkey");
  hashPartitions.add("timestamprowkey");
  CreateTableOptions thisTableOptions = new CreateTableOptions()
      .setNumReplicas(1)
      .addHashPartitions(hashPartitions,HASH_BUCKETS_SIZE_FOR_ALL_HASH_COL)
      .setRangePartitionColumns(rangeKeys);
  int stepsize = Integer.MAX_VALUE / SPLIT_COUNT_FOR_INT_ROW_KEY;
  int splitBoundary = stepsize;
  Schema schema = buildSchemaForUnitTestsTable();
  for ( int i = 0; i < SPLIT_COUNT_FOR_INT_ROW_KEY; i++) {
    PartialRow splitRowBoundary = schema.newPartialRow();
    splitRowBoundary.addInt("introwkey",splitBoundary);
    thisTableOptions = thisTableOptions.addSplitRow(splitRowBoundary);
    splitBoundary += stepsize;
  }
  try {
    client.createTable(tableName, schema,thisTableOptions);
  } catch (KuduException e) {
    LOG.error("Error while creating table for unit tests " + e.getMessage(), e);
    throw e;
  }

}
 

private void createKuduTable() throws KuduException {
    KuduClient client =  harness.getClient();
    List<ColumnSchema> columns = new ArrayList<>();
    columns.add(new ColumnSchema.ColumnSchemaBuilder("id", Type.INT32).key(true).build());
    columns.add(new ColumnSchema.ColumnSchemaBuilder("stringval", Type.STRING).build());
    columns.add(new ColumnSchema.ColumnSchemaBuilder("varcharval", Type.VARCHAR).typeAttributes(
            new ColumnTypeAttributes.ColumnTypeAttributesBuilder().length(256).build()
    ).build());
    columns.add(new ColumnSchema.ColumnSchemaBuilder("num32val", Type.INT32).build());
    columns.add(new ColumnSchema.ColumnSchemaBuilder("timestampval", Type.UNIXTIME_MICROS).build());
    Schema schema = new Schema(columns);
    CreateTableOptions opts = new CreateTableOptions()
        .addHashPartitions(Collections.singletonList("id"), 4);
    client.createTable(DEFAULT_TABLE_NAME, schema, opts);
}
 

public static KuduTableInfo createTableInfo(String tableName, TableSchema schema, Map<String, String> props) {
    // Since KUDU_HASH_COLS is a required property for table creation, we use it to infer whether to create table
    boolean createIfMissing = props.containsKey(KUDU_HASH_COLS);

    KuduTableInfo tableInfo = KuduTableInfo.forTable(tableName);

    if (createIfMissing) {

        List<Tuple2<String, DataType>> columns = getSchemaWithSqlTimestamp(schema)
                .getTableColumns()
                .stream()
                .map(tc -> Tuple2.of(tc.getName(), tc.getType()))
                .collect(Collectors.toList());

        List<String> keyColumns = getPrimaryKeyColumns(props, schema);
        ColumnSchemasFactory schemasFactory = () -> toKuduConnectorColumns(columns, keyColumns);
        List<String> hashColumns = getHashColumns(props);
        int replicas = Optional.ofNullable(props.get(KuduTableFactory.KUDU_REPLICAS)).map(Integer::parseInt).orElse(1);

        CreateTableOptionsFactory optionsFactory = () -> new CreateTableOptions()
                .setNumReplicas(replicas)
                .addHashPartitions(hashColumns, replicas * 2);

        tableInfo.createTableIfNotExists(schemasFactory, optionsFactory);
    } else {
        LOG.debug("Property {} is missing, assuming the table is already created.", KUDU_HASH_COLS);
    }

    return tableInfo;
}
 

private CreateTableOptions buildCreateTableOptions(Schema schema, Map<String, Object> properties) {
    CreateTableOptions options = new CreateTableOptions();

    RangePartitionDefinition rangePartitionDefinition = null;
    Optional<PartitionDesign> optPartitionDesign = KuduTableProperties.getPartitionDesign(properties);
    if (optPartitionDesign.isPresent()) {
        PartitionDesign partitionDesign = optPartitionDesign.get();
        if (partitionDesign.getHash() != null) {
            for (HashPartitionDefinition partition : partitionDesign.getHash()) {
                options.addHashPartitions(partition.getColumns(), partition.getBuckets());
            }
        }
        if (partitionDesign.getRange() != null) {
            rangePartitionDefinition = partitionDesign.getRange();
            options.setRangePartitionColumns(rangePartitionDefinition.getColumns());
        }
    } else {
        String firstColumn = schema.getColumnByIndex(0).getName();
        options.setRangePartitionColumns(Collections.singletonList(firstColumn));
    }

    List<RangePartition> rangePartitions = KuduTableProperties.getRangePartitions(properties);
    if (rangePartitionDefinition != null && !rangePartitions.isEmpty()) {
        for (RangePartition rangePartition: rangePartitions) {
            PartialRow lower = KuduTableProperties.toRangeBoundToPartialRow(schema, rangePartitionDefinition, rangePartition.getLower());
            PartialRow upper = KuduTableProperties.toRangeBoundToPartialRow(schema, rangePartitionDefinition, rangePartition.getUpper());
            options.addRangePartition(lower, upper);
        }
    }


    Optional<Integer> numReplicas = KuduTableProperties.getNumReplicas(properties);
    numReplicas.ifPresent(options::setNumReplicas);

    return options;
}
 

@Test
public void testBeforeProducerFromModel() throws Exception {
    customizer.customize(getComponent(), new HashMap<>());

    KuduTable model = new KuduTable();
    model.setName("KuduTestTable");

    KuduTable.Schema modelSchema = new KuduTable.Schema();
    KuduTable.ColumnSchema[] modelSchemaColumns = {
            new KuduTable.ColumnSchema("id", "Integer", true),
            new KuduTable.ColumnSchema("title", "String", false),
            new KuduTable.ColumnSchema("name", "String", false),
            new KuduTable.ColumnSchema("lastname", "String", false)
    };

    modelSchema.setColumns(modelSchemaColumns, true);
    model.setSchema(modelSchema);

    Exchange inbound = new DefaultExchange(createCamelContext());
    inbound.getIn().setBody(model);
    getComponent().getBeforeProducer().process(inbound);

    Schema schema = (Schema) inbound.getIn().getHeader("Schema");
    CreateTableOptions builder = (CreateTableOptions) inbound.getIn().getHeader("TableOptions");

    Assert.assertNotNull(schema);
    Assert.assertNotNull(builder);

    Assert.assertEquals("Table schema has all elements", 4, schema.getColumnCount());
    Assert.assertEquals("Name of the first column matches", "id", schema.getColumn("id").getName());
    Assert.assertEquals("Type of the first column matches", "int32", schema.getColumn("id").getType().getName());

    Assert.assertEquals("Name of the first column matches", "name", schema.getColumn("name").getName());
    Assert.assertEquals("Type of the first column matches", "string", schema.getColumn("name").getType().getName());
}
 

@Ignore
public void createTable() throws InterruptedException, KuduException {
    deleteTestTable(TABLE, HOST + ":" + PORT);

    errorEndpoint.expectedMessageCount(0);
    successEndpoint.expectedMessageCount(1);

    final Map<String, Object> headers = new HashMap<>();

    final List<ColumnSchema> columns = new ArrayList<>(5);
    final List<String> columnNames = Arrays.asList("id", "title", "name", "lastname", "address");

    for (int i = 0; i < columnNames.size(); i++) {
        columns.add(
            new ColumnSchema.ColumnSchemaBuilder(columnNames.get(i), Type.STRING)
                .key(i == 0)
                .build());
    }

    final List<String> rangeKeys = new ArrayList<>();
    rangeKeys.add("id");

    headers.put("Schema", new Schema(columns));
    headers.put("TableOptions", new CreateTableOptions().setRangePartitionColumns(rangeKeys));

    requestBodyAndHeaders("direct://create", null, headers);

    errorEndpoint.assertIsSatisfied();
    successEndpoint.assertIsSatisfied();
}
 

private CreateTableOptions buildCreateTableOptions(Schema schema, Map<String, Object> properties)
{
    CreateTableOptions options = new CreateTableOptions();

    RangePartitionDefinition rangePartitionDefinition = null;
    PartitionDesign partitionDesign = KuduTableProperties.getPartitionDesign(properties);
    if (partitionDesign.getHash() != null) {
        for (HashPartitionDefinition partition : partitionDesign.getHash()) {
            options.addHashPartitions(partition.getColumns(), partition.getBuckets());
        }
    }
    if (partitionDesign.getRange() != null) {
        rangePartitionDefinition = partitionDesign.getRange();
        options.setRangePartitionColumns(rangePartitionDefinition.getColumns());
    }

    List<RangePartition> rangePartitions = KuduTableProperties.getRangePartitions(properties);
    if (rangePartitionDefinition != null && !rangePartitions.isEmpty()) {
        for (RangePartition rangePartition : rangePartitions) {
            PartialRow lower = KuduTableProperties.toRangeBoundToPartialRow(schema, rangePartitionDefinition, rangePartition.getLower());
            PartialRow upper = KuduTableProperties.toRangeBoundToPartialRow(schema, rangePartitionDefinition, rangePartition.getUpper());
            options.addRangePartition(lower, upper);
        }
    }

    Optional<Integer> numReplicas = KuduTableProperties.getNumReplicas(properties);
    numReplicas.ifPresent(options::setNumReplicas);

    return options;
}
 

@Test
void testSpeed() throws Exception {
    String masterAddresses = harness.getMasterAddressesAsString();

    KuduTableInfo tableInfo = KuduTableInfo
            .forTable("test_speed")
            .createTableIfNotExists(
                    () ->
                            Lists.newArrayList(
                                    new ColumnSchema
                                            .ColumnSchemaBuilder("id", Type.INT32)
                                            .key(true)
                                            .build(),
                                    new ColumnSchema
                                            .ColumnSchemaBuilder("uuid", Type.STRING)
                                            .build()
                            ),
                    () -> new CreateTableOptions()
                            .setNumReplicas(3)
                            .addHashPartitions(Lists.newArrayList("id"), 6));

    KuduWriterConfig writerConfig = KuduWriterConfig.Builder
            .setMasters(masterAddresses)
            .setEventualConsistency()
            .build();
    KuduSink<Row> sink = new KuduSink<>(writerConfig, tableInfo, new RowOperationMapper(columns, AbstractSingleOperationMapper.KuduOperation.INSERT));

    sink.setRuntimeContext(context);
    sink.open(new Configuration());

    int totalRecords = 100000;
    for (int i = 0; i < totalRecords; i++) {
        Row kuduRow = new Row(2);
        kuduRow.setField(0, i);
        kuduRow.setField(1, UUID.randomUUID().toString());
        sink.invoke(kuduRow);
    }

    // sleep to allow eventual consistency to finish
    Thread.sleep(1000);

    sink.close();

    List<Row> rows = readRows(tableInfo);
    Assertions.assertEquals(totalRecords, rows.size());
}
 

static CreateTableOptions createTableOptions() {
  return new CreateTableOptions()
      .setRangePartitionColumns(ImmutableList.of(COL_ID))
      .setNumReplicas(1);
}
 

@Test
public void dataStreamEndToEstTest() throws Exception {
    KuduCatalog catalog = new KuduCatalog(harness.getMasterAddressesAsString());
    // Creating table through catalog
    KuduTableFactory tableFactory = catalog.getKuduTableFactory();

    KuduTableInfo tableInfo = KuduTableInfo.forTable("TestTable7").createTableIfNotExists(
            () ->
                    Lists.newArrayList(
                            new ColumnSchema
                                    .ColumnSchemaBuilder("k", Type.INT32)
                                    .key(true)
                                    .build(),
                            new ColumnSchema
                                    .ColumnSchemaBuilder("v", Type.STRING)
                                    .build()
                    ),
            () -> new CreateTableOptions()
                    .setNumReplicas(1)
                    .addHashPartitions(Lists.newArrayList("k"), 2));

    catalog.createTable(tableInfo, false);

    ObjectPath path = catalog.getObjectPath("TestTable7");
    CatalogTable table = catalog.getTable(path);

    List<Tuple2<Integer, String>> input = Lists.newArrayList(Tuple2.of(1, "one"), Tuple2.of(2, "two"), Tuple2.of(3, "three"));

    // Writing with simple sink
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setParallelism(2);
    KuduWriterConfig writerConfig = KuduWriterConfig.Builder.setMasters(harness.getMasterAddressesAsString()).build();
    env.fromCollection(input).addSink(
            new KuduSink<>(
                    writerConfig,
                    tableInfo,
                    new TupleOperationMapper<>(
                            new String[]{"k", "v"},
                            AbstractSingleOperationMapper.KuduOperation.INSERT)
            )
    );
    env.execute();
    // Reading and validating data
    env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setParallelism(2);

    CollectionSink.output.clear();
    tableFactory.createTableSource(path, table)
            .getDataStream(env)
            .map(row -> Tuple2.of((int) row.getField(0), (String) row.getField(1)))
            .returns(new TypeHint<Tuple2<Integer, String>>() {
            })
            .addSink(new CollectionSink<>()).setParallelism(1);
    env.execute();

    List<Tuple2<Integer, String>> expected = Lists.newArrayList(Tuple2.of(1, "one"), Tuple2.of(2, "two"), Tuple2.of(3, "three"));
    assertEquals(new HashSet<>(expected), new HashSet<>(CollectionSink.output));
    CollectionSink.output.clear();
}
 

@Before
public void init() throws Exception {
    testRunner = TestRunners.newTestRunner(SampleProcessor.class);
    testRunner.setValidateExpressionUsage(false);
    final String tableName = "table1";

    KuduClient client =  harness.getClient();
    List<ColumnSchema> columns = new ArrayList<>();
    columns.add(new ColumnSchema.ColumnSchemaBuilder("string", Type.STRING).key(true).build());
    columns.add(new ColumnSchema.ColumnSchemaBuilder("binary", Type.BINARY).build());
    columns.add(new ColumnSchema.ColumnSchemaBuilder("bool", Type.BOOL).build());
    columns.add(new ColumnSchema
            .ColumnSchemaBuilder("decimal", Type.DECIMAL)
            .typeAttributes(DecimalUtil.typeAttributes(DecimalUtil.MAX_DECIMAL64_PRECISION, 1))
            .build()
    );
    columns.add(new ColumnSchema.ColumnSchemaBuilder("double", Type.DOUBLE).build());
    columns.add(new ColumnSchema.ColumnSchemaBuilder("float", Type.FLOAT).build());
    columns.add(new ColumnSchema.ColumnSchemaBuilder("int8", Type.INT8).build());
    columns.add(new ColumnSchema.ColumnSchemaBuilder("int16", Type.INT16).build());
    columns.add(new ColumnSchema.ColumnSchemaBuilder("int32", Type.INT32).build());
    columns.add(new ColumnSchema.ColumnSchemaBuilder("int64", Type.INT64).build());
    columns.add(new ColumnSchema.ColumnSchemaBuilder("unixtime_micros", Type.UNIXTIME_MICROS).build());
    columns.add(new ColumnSchema.ColumnSchemaBuilder("varchar_3", Type.VARCHAR).typeAttributes(
            new ColumnTypeAttributes.ColumnTypeAttributesBuilder().length(3).build()
    ).build());
    Schema schema = new Schema(columns);

    CreateTableOptions opts = new CreateTableOptions().setRangePartitionColumns(Collections.singletonList("string"));
    client.createTable(tableName, schema, opts);

    KuduTable table = client.openTable(tableName);
    KuduSession session = client.newSession();

    Insert insert = table.newInsert();
    PartialRow row = insert.getRow();
    row.addString("string", "string1");
    row.addBinary("binary", "binary1".getBytes());
    row.addBoolean("bool",true);
    row.addDecimal("decimal", BigDecimal.valueOf(0.1));
    row.addDouble("double",0.2);
    row.addFloat("float",0.3f);
    row.addByte("int8", (byte) 1);
    row.addShort("int16", (short) 2);
    row.addInt("int32",3);
    row.addLong("int64",4L);
    row.addTimestamp("unixtime_micros", new Timestamp(nowMillis));
    row.addVarchar("varchar_3", "SFO");
    session.apply(insert);

    insert = table.newInsert();
    row = insert.getRow();
    row.addString("string", "string2");
    row.addBinary("binary", "binary2".getBytes());
    row.addBoolean("bool",false);
    row.addDecimal("decimal", BigDecimal.valueOf(0.1));
    row.addDouble("double",1.2);
    row.addFloat("float",1.3f);
    row.addByte("int8", (byte) 11);
    row.addShort("int16", (short) 12);
    row.addInt("int32",13);
    row.addLong("int64",14L);
    row.addTimestamp("unixtime_micros", new Timestamp(nowMillis+(1000L * 60 * 60 * 24 * 365))); //+ 1 year
    row.addVarchar("varchar_3", "SJC");
    session.apply(insert);

    session.close();

    kuduLookupService = new KuduLookupService();
    testRunner.addControllerService("kuduLookupService", kuduLookupService);
    testRunner.setProperty(kuduLookupService, KuduLookupService.KUDU_MASTERS, "testLocalHost:7051");
    testRunner.setProperty(kuduLookupService, KuduLookupService.KUDU_REPLICA_SELECTION, KuduLookupService.LEADER_ONLY);
    testRunner.setProperty(kuduLookupService, KuduLookupService.TABLE_NAME, tableName);
    kuduLookupService.kuduClient = client;
}
 

/**
 * Creates the {@link CreateTableOptions} that will be used during the createTable operation.
 *
 * @return CreateTableOptions for creating the table.
 */
CreateTableOptions getCreateTableOptions();