Java Code Examples for org.apache.kudu.client.CreateTableOptions#addHashPartitions()

The following examples show how to use org.apache.kudu.client.CreateTableOptions#addHashPartitions() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: SchemaEmulationByTableNameConvention.java    From presto with Apache License 2.0 6 votes vote down vote up 
private void createAndFillSchemasTable(KuduClient client)
        throws KuduException
{
    List<String> existingSchemaNames = listSchemaNamesFromTablets(client);
    ColumnSchema schemaColumnSchema = new ColumnSchema.ColumnSchemaBuilder("schema", Type.STRING)
            .key(true).build();
    Schema schema = new Schema(ImmutableList.of(schemaColumnSchema));
    CreateTableOptions options = new CreateTableOptions();
    options.addHashPartitions(ImmutableList.of(schemaColumnSchema.getName()), 2);
    KuduTable schemasTable = client.createTable(rawSchemasTableName, schema, options);
    KuduSession session = client.newSession();
    try {
        session.setFlushMode(SessionConfiguration.FlushMode.AUTO_FLUSH_BACKGROUND);
        for (String schemaName : existingSchemaNames) {
            Insert insert = schemasTable.newInsert();
            insert.getRow().addString(0, schemaName);
            session.apply(insert);
        }
    }
    finally {
        session.close();
    }
}
Example 2
Source File: KuduTableInfo.java    From flink-learning with Apache License 2.0 6 votes vote down vote up 
public CreateTableOptions getCreateTableOptions() {
    CreateTableOptions options = new CreateTableOptions();
    if(replicas!=null){
        options.setNumReplicas(replicas);
    }
    if(hasColummns()) {
        List<String> rangeKeys = new ArrayList<>();
        List<String> hashKeys = new ArrayList<>();
        for(KuduColumnInfo column : columns){
            if(column.isRangeKey()){
                rangeKeys.add(column.name());
            }
            if(column.isHashKey()){
                hashKeys.add(column.name());
            }
        }
        options.setRangePartitionColumns(rangeKeys);
        options.addHashPartitions(hashKeys, replicas*2);
    }

    return options;
}
Example 3
Source File: KuduTableInfo.java    From flink-learning with Apache License 2.0 6 votes vote down vote up 
public CreateTableOptions getCreateTableOptions() {
    CreateTableOptions options = new CreateTableOptions();
    if(replicas!=null){
        options.setNumReplicas(replicas);
    }
    if(hasColummns()) {
        List<String> rangeKeys = new ArrayList<>();
        List<String> hashKeys = new ArrayList<>();
        for(KuduColumnInfo column : columns){
            if(column.isRangeKey()){
                rangeKeys.add(column.name());
            }
            if(column.isHashKey()){
                hashKeys.add(column.name());
            }
        }
        options.setRangePartitionColumns(rangeKeys);
        options.addHashPartitions(hashKeys, replicas*2);
    }

    return options;
}
Example 4
Source File: NativeKuduClientSession.java    From presto-kudu with Apache License 2.0 6 votes vote down vote up 
private void createAndFillSchemasTable() throws KuduException {
    List<String> existingSchemaNames = listSchemaNamesFromTablets();
    ColumnSchema tenantColumnSchema = new ColumnSchema.ColumnSchemaBuilder("tenant", Type.STRING)
            .key(true).build();
    ColumnSchema schemaColumnSchema = new ColumnSchema.ColumnSchemaBuilder("schema", Type.STRING)
            .key(true).build();
    Schema schema = new Schema(ImmutableList.of(tenantColumnSchema, schemaColumnSchema));
    CreateTableOptions options = new CreateTableOptions();
    options.setNumReplicas(1); // TODO config
    options.addHashPartitions(ImmutableList.of(tenantColumnSchema.getName()), 2);
    KuduTable schemasTable = client.createTable(rawSchemasTableName, schema, options);
    KuduSession session = client.newSession();
    session.setFlushMode(SessionConfiguration.FlushMode.AUTO_FLUSH_BACKGROUND);
    try {
        for (String schemaName : existingSchemaNames) {
            Insert insert = schemasTable.newInsert();
            fillSchemaRow(insert.getRow(), schemaName);
            session.apply(insert);
        }
    } finally {
        session.close();
    }
}
Example 5
Source File: KuduClientSession.java    From presto with Apache License 2.0 5 votes vote down vote up 
private CreateTableOptions buildCreateTableOptions(Schema schema, Map<String, Object> properties)
{
    CreateTableOptions options = new CreateTableOptions();

    RangePartitionDefinition rangePartitionDefinition = null;
    PartitionDesign partitionDesign = KuduTableProperties.getPartitionDesign(properties);
    if (partitionDesign.getHash() != null) {
        for (HashPartitionDefinition partition : partitionDesign.getHash()) {
            options.addHashPartitions(partition.getColumns(), partition.getBuckets());
        }
    }
    if (partitionDesign.getRange() != null) {
        rangePartitionDefinition = partitionDesign.getRange();
        options.setRangePartitionColumns(rangePartitionDefinition.getColumns());
    }

    List<RangePartition> rangePartitions = KuduTableProperties.getRangePartitions(properties);
    if (rangePartitionDefinition != null && !rangePartitions.isEmpty()) {
        for (RangePartition rangePartition : rangePartitions) {
            PartialRow lower = KuduTableProperties.toRangeBoundToPartialRow(schema, rangePartitionDefinition, rangePartition.getLower());
            PartialRow upper = KuduTableProperties.toRangeBoundToPartialRow(schema, rangePartitionDefinition, rangePartition.getUpper());
            options.addRangePartition(lower, upper);
        }
    }

    Optional<Integer> numReplicas = KuduTableProperties.getNumReplicas(properties);
    numReplicas.ifPresent(options::setNumReplicas);

    return options;
}
Example 6
Source File: NativeKuduClientSession.java    From presto-kudu with Apache License 2.0 5 votes vote down vote up 
private CreateTableOptions buildCreateTableOptions(Schema schema, Map<String, Object> properties) {
    CreateTableOptions options = new CreateTableOptions();

    RangePartitionDefinition rangePartitionDefinition = null;
    Optional<PartitionDesign> optPartitionDesign = KuduTableProperties.getPartitionDesign(properties);
    if (optPartitionDesign.isPresent()) {
        PartitionDesign partitionDesign = optPartitionDesign.get();
        if (partitionDesign.getHash() != null) {
            for (HashPartitionDefinition partition : partitionDesign.getHash()) {
                options.addHashPartitions(partition.getColumns(), partition.getBuckets());
            }
        }
        if (partitionDesign.getRange() != null) {
            rangePartitionDefinition = partitionDesign.getRange();
            options.setRangePartitionColumns(rangePartitionDefinition.getColumns());
        }
    } else {
        String firstColumn = schema.getColumnByIndex(0).getName();
        options.setRangePartitionColumns(Collections.singletonList(firstColumn));
    }

    List<RangePartition> rangePartitions = KuduTableProperties.getRangePartitions(properties);
    if (rangePartitionDefinition != null && !rangePartitions.isEmpty()) {
        for (RangePartition rangePartition: rangePartitions) {
            PartialRow lower = KuduTableProperties.toRangeBoundToPartialRow(schema, rangePartitionDefinition, rangePartition.getLower());
            PartialRow upper = KuduTableProperties.toRangeBoundToPartialRow(schema, rangePartitionDefinition, rangePartition.getUpper());
            options.addRangePartition(lower, upper);
        }
    }


    Optional<Integer> numReplicas = KuduTableProperties.getNumReplicas(properties);
    numReplicas.ifPresent(options::setNumReplicas);

    return options;
}
Example 7
Source File: Tables.java    From kudu-ts with Apache License 2.0 5 votes vote down vote up 
/**
 * The {@code metrics} table is hash partitioned on the {@code metric} and
 * {@code tagset_id} columns, and range partitioned on the {@code time}
 * column. The hash partitioning allows writes and scans at the current time
 * to be evenly distributed over the cluster. Range partitioning on time
 * allows whole tablets to be pruned based on the time constraint, and allows
 * old data to be dropped if desired.
 * @param options the create options
 * @param numTabletServers the number of tablet servers
 * @return the tags table create options
 */
static CreateTableOptions metricsCreateTableOptions(CreateOptions options,
                                                    int numTabletServers) {
  CreateTableOptions create = new CreateTableOptions();
  create.setNumReplicas(options.getNumReplicas());
  create.addHashPartitions(ImmutableList.of("metric", "tagset_id"),
                           options.getNumMetricsHashBuckets(numTabletServers));
  create.setRangePartitionColumns(ImmutableList.of("time"));
  for (Long time : options.getMetricsSplits()) {
    PartialRow split = METRICS_SCHEMA.newPartialRow();
    split.addLong(METRICS_TIME_INDEX, time);
    create.addSplitRow(split);
  }
  return create;
}
Example 8
Source File: Tables.java    From kudu-ts with Apache License 2.0 5 votes vote down vote up 
/**
 * The {@code tags} table is hash partitioned on the {@code key} and
 * {@code value} columns, which allows a scan of a ({@code key}, {@code value})
 * pair to be fully satisfied by a single tablet, and gives good protection
 * against hotspotting.
 * @param options the create options
 * @param numTabletServers the number of tablet servers
 * @return the tags table create options
 */
static CreateTableOptions tagsCreateTableOptions(CreateOptions options,
                                                 int numTabletServers) {
  CreateTableOptions create = new CreateTableOptions();
  create.setNumReplicas(options.getNumReplicas());
  create.addHashPartitions(ImmutableList.of("key", "value"),
                           options.getNumTagsTablets(numTabletServers));
  create.setRangePartitionColumns(ImmutableList.<String>of());
  return create;
}