org.apache.kudu.Schema Java Examples

The following examples show how to use org.apache.kudu.Schema. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: KuduClientSession.java    From presto with Apache License 2.0 7 votes vote down vote up
public KuduTable createTable(ConnectorTableMetadata tableMetadata, boolean ignoreExisting)
{
    try {
        String rawName = schemaEmulation.toRawName(tableMetadata.getTable());
        if (ignoreExisting) {
            if (client.tableExists(rawName)) {
                return null;
            }
        }

        if (!schemaEmulation.existsSchema(client, tableMetadata.getTable().getSchemaName())) {
            throw new SchemaNotFoundException(tableMetadata.getTable().getSchemaName());
        }

        List<ColumnMetadata> columns = tableMetadata.getColumns();
        Map<String, Object> properties = tableMetadata.getProperties();

        Schema schema = buildSchema(columns, properties);
        CreateTableOptions options = buildCreateTableOptions(schema, properties);
        return client.createTable(rawName, schema, options);
    }
    catch (KuduException e) {
        throw new PrestoException(GENERIC_INTERNAL_ERROR, e);
    }
}
 
Example #2
Source File: NativeKuduClientSession.java    From presto-kudu with Apache License 2.0 6 votes vote down vote up
@Override
public KuduTable createTable(ConnectorTableMetadata tableMetadata, boolean ignoreExisting) {
    try {
        SchemaTableName schemeTableName= tableMetadata.getTable();
        String rawName = toRawName(schemeTableName);
        if (ignoreExisting) {
            if (client.tableExists(rawName)) {
                return null;
            }
        }
        if(!schemaExists(schemeTableName.getSchemaName())){
            throw new SchemaNotFoundException(schemeTableName.getSchemaName());
        }
        List<ColumnMetadata> columns = tableMetadata.getColumns();
        Map<String, Object> properties = tableMetadata.getProperties();

        Schema schema = buildSchema(columns, properties);
        CreateTableOptions options = buildCreateTableOptions(schema, properties);
        return client.createTable(rawName, schema, options);
    } catch (KuduException e) {
        throw new PrestoException(GENERIC_INTERNAL_ERROR, e);
    }
}
 
Example #3
Source File: KuduTableProperties.java    From presto with Apache License 2.0 6 votes vote down vote up
private static Object toValue(Schema schema, PartialRow bound, Integer idx)
{
    Type type = schema.getColumnByIndex(idx).getType();
    switch (type) {
        case UNIXTIME_MICROS:
            long millis = bound.getLong(idx) / 1000;
            return ISODateTimeFormat.dateTime().withZone(DateTimeZone.UTC).print(millis);
        case STRING:
            return bound.getString(idx);
        case INT64:
            return bound.getLong(idx);
        case INT32:
            return bound.getInt(idx);
        case INT16:
            return bound.getShort(idx);
        case INT8:
            return (short) bound.getByte(idx);
        case BOOL:
            return bound.getBoolean(idx);
        case BINARY:
            return bound.getBinaryCopy(idx);
        default:
            throw new IllegalStateException("Unhandled type " + type + " for range partition");
    }
}
 
Example #4
Source File: KuduTableProperties.java    From presto with Apache License 2.0 6 votes vote down vote up
private static RangeBoundValue buildRangePartitionBound(KuduTable table, byte[] rangeKey)
{
    if (rangeKey.length == 0) {
        return null;
    }
    else {
        Schema schema = table.getSchema();
        PartitionSchema partitionSchema = table.getPartitionSchema();
        PartitionSchema.RangeSchema rangeSchema = partitionSchema.getRangeSchema();
        List<Integer> rangeColumns = rangeSchema.getColumns();

        final int numColumns = rangeColumns.size();

        PartialRow bound = KeyEncoderAccessor.decodeRangePartitionKey(schema, partitionSchema, rangeKey);

        ArrayList<Object> list = new ArrayList<>();
        for (int i = 0; i < numColumns; i++) {
            Object obj = toValue(schema, bound, rangeColumns.get(i));
            list.add(obj);
        }
        return new RangeBoundValue(list);
    }
}
 
Example #5
Source File: KuduCatalogTest.java    From bahir-flink with Apache License 2.0 6 votes vote down vote up
private void validateMultiKey(String tableName) throws Exception {
    KuduTable kuduTable = harness.getClient().openTable(tableName);
    Schema schema = kuduTable.getSchema();

    assertEquals(2, schema.getPrimaryKeyColumnCount());
    assertEquals(3, schema.getColumnCount());

    assertTrue(schema.getColumn("first").isKey());
    assertTrue(schema.getColumn("second").isKey());

    assertFalse(schema.getColumn("third").isKey());

    KuduScanner scanner = harness.getClient().newScannerBuilder(kuduTable).build();
    List<RowResult> rows = new ArrayList<>();
    scanner.forEach(rows::add);

    assertEquals(1, rows.size());
    assertEquals("f", rows.get(0).getString("first"));
    assertEquals(2, rows.get(0).getInt("second"));
    assertEquals("t", rows.get(0).getString("third"));
}
 
Example #6
Source File: KuduTableProperties.java    From presto with Apache License 2.0 6 votes vote down vote up
public static PartialRow toRangeBoundToPartialRow(Schema schema, RangePartitionDefinition definition,
        RangeBoundValue boundValue)
{
    PartialRow partialRow = new PartialRow(schema);
    if (boundValue != null) {
        List<Integer> rangeColumns = definition.getColumns().stream()
                .map(schema::getColumnIndex).collect(toImmutableList());

        if (rangeColumns.size() != boundValue.getValues().size()) {
            throw new IllegalStateException("Expected " + rangeColumns.size()
                    + " range columns, but got " + boundValue.getValues().size());
        }
        for (int i = 0; i < rangeColumns.size(); i++) {
            Object obj = boundValue.getValues().get(i);
            int idx = rangeColumns.get(i);
            ColumnSchema columnSchema = schema.getColumnByIndex(idx);
            setColumnValue(partialRow, idx, obj, columnSchema.getType(), columnSchema.getName());
        }
    }
    return partialRow;
}
 
Example #7
Source File: KuduMetadata.java    From presto-kudu with Apache License 2.0 6 votes vote down vote up
@Override
public Map<String, ColumnHandle> getColumnHandles(ConnectorSession session,
                                                  ConnectorTableHandle connectorTableHandle) {
    KuduTableHandle tableHandle = fromConnectorTableHandle(session, connectorTableHandle);
    Schema schema = clientSession.getTableSchema(tableHandle);

    ImmutableMap.Builder<String, ColumnHandle> columnHandles = ImmutableMap.builder();
    for (int i = 0; i < schema.getColumnCount(); i++) {
        ColumnSchema col = schema.getColumnByIndex(i);
        String name = col.getName();
        Type type = TypeHelper.fromKuduColumn(col);
        KuduColumnHandle columnHandle = new KuduColumnHandle(name, i, type);
        columnHandles.put(name, columnHandle);
    }

    return columnHandles.build();
}
 
Example #8
Source File: KuduServiceImpl.java    From beam with Apache License 2.0 6 votes vote down vote up
/** Configures the scanner builder to conform to the spec. */
private static <T2> void configureBuilder(
    KuduIO.Read<T2> spec, Schema schema, AbstractKuduScannerBuilder builder) {
  builder.cacheBlocks(true); // as per kudu-spark
  if (spec.getBatchSize() != null) {
    builder.batchSizeBytes(spec.getBatchSize());
  }
  if (spec.getProjectedColumns() != null) {
    builder.setProjectedColumnNames(spec.getProjectedColumns());
  }
  if (spec.getFaultTolerent() != null) {
    builder.setFaultTolerant(spec.getFaultTolerent());
  }
  if (spec.getSerializablePredicates() != null) {
    for (Common.ColumnPredicatePB predicate : spec.getSerializablePredicates()) {
      builder.addPredicate(KuduPredicate.fromPB(schema, predicate));
    }
  }
}
 
Example #9
Source File: KuduCreateTableCustomizerTest.java    From syndesis with Apache License 2.0 6 votes vote down vote up
@Test
public void testBeforeProducerFromOptions() throws Exception {
    Map<String, Object> options = new HashMap<>();
    options.put("columns", "Integer,id;String,title;String,name;String,lastname");

    customizer.customize(getComponent(), options);

    Exchange inbound = new DefaultExchange(createCamelContext());
    getComponent().getBeforeProducer().process(inbound);

    Schema schema = (Schema) inbound.getIn().getHeader("Schema");
    CreateTableOptions builder = (CreateTableOptions) inbound.getIn().getHeader("TableOptions");

    Assert.assertNotNull(schema);
    Assert.assertNotNull(builder);

    Assert.assertEquals("Table schema has all elements", 4, schema.getColumnCount());
    Assert.assertEquals("Name of the first column matches", "id", schema.getColumn("id").getName());
    Assert.assertEquals("Type of the first column matches", "int32", schema.getColumn("id").getType().getName());

    Assert.assertEquals("Name of the first column matches", "name", schema.getColumn("name").getName());
    Assert.assertEquals("Type of the first column matches", "string", schema.getColumn("name").getType().getName());
}
 
Example #10
Source File: KuduMetadata.java    From presto with Apache License 2.0 6 votes vote down vote up
@Override
public ConnectorInsertTableHandle beginInsert(ConnectorSession session, ConnectorTableHandle connectorTableHandle)
{
    KuduTableHandle tableHandle = (KuduTableHandle) connectorTableHandle;

    KuduTable table = tableHandle.getTable(clientSession);
    Schema schema = table.getSchema();

    List<ColumnSchema> columns = schema.getColumns();
    List<Type> columnTypes = columns.stream()
            .map(TypeHelper::fromKuduColumn).collect(toImmutableList());

    return new KuduInsertTableHandle(
            tableHandle.getSchemaTableName(),
            columnTypes,
            table);
}
 
Example #11
Source File: TestPutKudu.java    From nifi with Apache License 2.0 6 votes vote down vote up
@Test
public void testAddingMissingFieldsWhenHandleSchemaDriftIsAllowed() throws InitializationException, IOException {
    // given
    processor.setTableSchema(new Schema(Arrays.asList()));
    createRecordReader(5);
    final String filename = "testAddingMissingFieldsWhenHandleSchemaDriftIsAllowed-" + System.currentTimeMillis();

    final Map<String,String> flowFileAttributes = new HashMap<>();
    flowFileAttributes.put(CoreAttributes.FILENAME.key(), filename);

    testRunner.setProperty(PutKudu.HANDLE_SCHEMA_DRIFT, "true");
    testRunner.enqueue("trigger", flowFileAttributes);

    // when
    testRunner.run();

    // then
    testRunner.assertAllFlowFilesTransferred(PutKudu.REL_SUCCESS, 1);
}
 
Example #12
Source File: KuduRecordSet.java    From presto with Apache License 2.0 6 votes vote down vote up
@Override
public RecordCursor cursor()
{
    KuduScanner scanner = clientSession.createScanner(kuduSplit);
    Schema projectedSchema = scanner.getProjectionSchema();
    ImmutableMap.Builder<Integer, Integer> builder = ImmutableMap.builder();
    for (int i = 0; i < columns.size(); i++) {
        KuduColumnHandle handle = (KuduColumnHandle) columns.get(i);
        if (handle.isVirtualRowId()) {
            builder.put(i, ROW_ID_POSITION);
        }
        else {
            builder.put(i, projectedSchema.getColumnIndex(handle.getName()));
        }
    }

    return new KuduRecordCursor(scanner, getTable(), getColumnTypes(), builder.build());
}
 
Example #13
Source File: KuduOperations.java    From geowave with Apache License 2.0 6 votes vote down vote up
@Override
public MetadataWriter createMetadataWriter(final MetadataType metadataType) {
  synchronized (CREATE_TABLE_MUTEX) {
    try {
      if (!metadataExists(metadataType)) {
        final List<ColumnSchema> columns = new ArrayList<>();
        for (final KuduMetadataField f : KuduMetadataField.values()) {
          f.addColumn(columns);
        }
        client.createTable(
            getKuduQualifiedName(getMetadataTableName(metadataType)),
            new Schema(columns),
            new CreateTableOptions().addHashPartitions(
                Collections.singletonList(KuduMetadataField.GW_PRIMARY_ID_KEY.getFieldName()),
                KuduUtils.KUDU_DEFAULT_BUCKETS).setNumReplicas(KuduUtils.KUDU_DEFAULT_REPLICAS));
      }
    } catch (final IOException e) {
      LOGGER.error(
          "Unable to create metadata table '{}'",
          getKuduQualifiedName(getMetadataTableName(metadataType)),
          e);
    }
  }
  return new KuduMetadataWriter(this, metadataType);
}
 
Example #14
Source File: KuduTS.java    From kudu-ts with Apache License 2.0 6 votes vote down vote up
private static Deferred<KuduTable> openOrCreateTable(final AsyncKuduClient client,
                                                     final String table,
                                                     final Schema schema,
                                                     final CreateTableOptions options) throws Exception {
  class CreateTableErrback implements Callback<Deferred<KuduTable>, Exception> {
    @Override
    public Deferred<KuduTable> call(Exception e) throws Exception {
      // TODO(danburkert): we should only do this if the error is "not found"
      LOG.debug("Creating table {}", table);
      return client.createTable(table, schema, options);
    }
    @Override
    public String toString() {
      return MoreObjects.toStringHelper(this).add("table", table).toString();
    }
  }

  return client.openTable(table).addErrback(new CreateTableErrback());
}
 
Example #15
Source File: AbstractKuduTest.java    From syndesis with Apache License 2.0 6 votes vote down vote up
protected void createTestTable(final String tableName, final String connection) throws KuduException {
    try (KuduClient client = new KuduClient.KuduClientBuilder(connection).build()) {

        final List<ColumnSchema> columns = new ArrayList<>(5);
        final List<String> columnNames = Arrays.asList("id", "title", "name", "lastname", "address");

        for (int i = 0; i < columnNames.size(); i++) {
            final Type type = i == 0 ? Type.INT32 : Type.STRING;
            columns.add(
                new ColumnSchema.ColumnSchemaBuilder(columnNames.get(i), type)
                    .key(i == 0)
                    .build());
        }

        final List<String> rangeKeys = new ArrayList<>();
        rangeKeys.add("id");

        client.createTable(tableName,
            new Schema(columns),
            new CreateTableOptions().setRangePartitionColumns(rangeKeys));
    }
}
 
Example #16
Source File: KuduCreateTableCustomizer.java    From syndesis with Apache License 2.0 6 votes vote down vote up
private void beforeProducer(Exchange exchange) {
    final Message in = exchange.getIn();
    final KuduTable model = exchange.getIn().getBody(KuduTable.class);

    if (model != null && ObjectHelper.isNotEmpty(model.getSchema())) {
        schema = model.getSchema();
    }

    KuduTable.ColumnSchema[] columnSchema = schema.getColumns();
    List<ColumnSchema> columns = new ArrayList<>(columnSchema.length);
    List<String> rangeKeys = new ArrayList<>();
    for (int i = 0; i < columnSchema.length; i++) {
        if (columnSchema[i].isKey()) {
            rangeKeys.add(columnSchema[i].getName());
        }

        columns.add(
                new ColumnSchema.ColumnSchemaBuilder(columnSchema[i].getName(), convertType(columnSchema[i].getType()))
                        .key(columnSchema[i].isKey())
                        .build()
        );
    }

    in.setHeader("Schema", new Schema(columns));
    in.setHeader("TableOptions", new CreateTableOptions().setRangePartitionColumns(rangeKeys));
}
 
Example #17
Source File: KuduUpdatablePageSource.java    From presto with Apache License 2.0 6 votes vote down vote up
@Override
public void deleteRows(Block rowIds)
{
    Schema schema = table.getSchema();
    KuduSession session = clientSession.newSession();
    session.setFlushMode(FlushMode.AUTO_FLUSH_BACKGROUND);
    try {
        try {
            for (int i = 0; i < rowIds.getPositionCount(); i++) {
                int len = rowIds.getSliceLength(i);
                Slice slice = rowIds.getSlice(i, 0, len);
                PartialRow row = KeyEncoderAccessor.decodePrimaryKey(schema, slice.getBytes());
                Delete delete = table.newDelete();
                RowHelper.copyPrimaryKey(schema, row, delete.getRow());
                session.apply(delete);
            }
        }
        finally {
            session.close();
        }
    }
    catch (KuduException e) {
        throw new RuntimeException(e);
    }
}
 
Example #18
Source File: SchemaEmulationByTableNameConvention.java    From presto with Apache License 2.0 6 votes vote down vote up
private void createAndFillSchemasTable(KuduClient client)
        throws KuduException
{
    List<String> existingSchemaNames = listSchemaNamesFromTablets(client);
    ColumnSchema schemaColumnSchema = new ColumnSchema.ColumnSchemaBuilder("schema", Type.STRING)
            .key(true).build();
    Schema schema = new Schema(ImmutableList.of(schemaColumnSchema));
    CreateTableOptions options = new CreateTableOptions();
    options.addHashPartitions(ImmutableList.of(schemaColumnSchema.getName()), 2);
    KuduTable schemasTable = client.createTable(rawSchemasTableName, schema, options);
    KuduSession session = client.newSession();
    try {
        session.setFlushMode(SessionConfiguration.FlushMode.AUTO_FLUSH_BACKGROUND);
        for (String schemaName : existingSchemaNames) {
            Insert insert = schemasTable.newInsert();
            insert.getRow().addString(0, schemaName);
            session.apply(insert);
        }
    }
    finally {
        session.close();
    }
}
 
Example #19
Source File: KuduResource.java    From camel-quarkus with Apache License 2.0 6 votes vote down vote up
@Path("/createTable")
@PUT
public Response createTable() {
    LOG.info("Calling createTable");

    final List<ColumnSchema> columns = new ArrayList<>(2);
    columns.add(new ColumnSchema.ColumnSchemaBuilder("id", Type.STRING).key(true).build());
    columns.add(new ColumnSchema.ColumnSchemaBuilder("name", Type.STRING).build());

    CreateTableOptions cto = new CreateTableOptions().setRangePartitionColumns(Arrays.asList("id")).setNumReplicas(1);

    final Map<String, Object> headers = new HashMap<>();
    headers.put(KuduConstants.CAMEL_KUDU_SCHEMA, new Schema(columns));
    headers.put(KuduConstants.CAMEL_KUDU_TABLE_OPTIONS, cto);

    producerTemplate.requestBodyAndHeaders("direct:create_table", null, headers);

    return Response.ok().build();
}
 
Example #20
Source File: KuduCreateTableCustomizer.java    From syndesis with Apache License 2.0 6 votes vote down vote up
private void setOptions(Map<String, Object> options) {
    if (options == null) {
        return;
    }

    if (!options.isEmpty()) {
        String[] columns = ConnectorOptions.extractOptionAndMap(options, "columns",
            names -> names.split(";", -1), new String[]{});
        KuduTable.ColumnSchema[] columnSchemas = new KuduTable.ColumnSchema[columns.length];

        for (int i = 0; i < columns.length; i++) {
            String[] column = columns[i].split(",", 2);
            columnSchemas[i] = new KuduTable.ColumnSchema(
                    column[1],
                    column[0],
                    i == 0
            );
        }

        schema = new KuduTable.Schema();
        schema.setColumns(columnSchemas, true);
    }

    options.put("operation", KuduDbOperations.CREATE_TABLE);
    options.put("type", KuduDbOperations.CREATE_TABLE);
}
 
Example #21
Source File: KuduUpdatablePageSource.java    From presto-kudu with Apache License 2.0 6 votes vote down vote up
@Override
public void deleteRows(Block rowIds) {
    Schema schema = table.getSchema();
    KuduSession session = clientSession.newSession();
    session.setFlushMode(FlushMode.AUTO_FLUSH_BACKGROUND);
    try {
        try {
            for (int i = 0; i < rowIds.getPositionCount(); i++) {
                int len = rowIds.getSliceLength(i);
                Slice slice = rowIds.getSlice(i, 0, len);
                PartialRow row = KeyEncoderAccessor.decodePrimaryKey(schema, slice.getBytes());
                Delete delete = table.newDelete();
                RowHelper.copyPrimaryKey(schema, row, delete.getRow());
                session.apply(delete);
            }
        } finally {
            session.close();
        }
    } catch (KuduException e) {
        throw new RuntimeException(e);
    }
}
 
Example #22
Source File: KuduTableProperties.java    From presto-kudu with Apache License 2.0 6 votes vote down vote up
public static PartialRow toRangeBoundToPartialRow(Schema schema, RangePartitionDefinition definition,
                                                  RangeBoundValue boundValue) {
    PartialRow partialRow = new PartialRow(schema);
    if (boundValue != null) {
        List<Integer> rangeColumns = definition.getColumns().stream()
                .map(name -> schema.getColumnIndex(name)).collect(toImmutableList());

        if (rangeColumns.size() != boundValue.getValues().size()) {
            throw new IllegalStateException("Expected " + rangeColumns.size()
                    + " range columns, but got " + boundValue.getValues().size());
        }
        for (int i = 0; i < rangeColumns.size(); i++) {
            Object obj = boundValue.getValues().get(i);
            int idx = rangeColumns.get(i);
            ColumnSchema columnSchema = schema.getColumnByIndex(idx);
            setColumnValue(partialRow, idx, obj, columnSchema.getType(), columnSchema.getName());
        }
    }
    return partialRow;
}
 
Example #23
Source File: KuduToBigQueryTest.java    From DataflowTemplates with Apache License 2.0 6 votes vote down vote up
/**
 * The schemas for the main input of the kudu rows transformation. Checking all possible supported
 * data types
 */
private Schema createTestKuduSchema() {
  List<ColumnSchema> columns =
      ImmutableList.<ColumnSchema>builder()
          .add(new ColumnSchema.ColumnSchemaBuilder("INT32", Type.INT32).key(true).build())
          .add(new ColumnSchema.ColumnSchemaBuilder("BOOL", Type.BOOL).nullable(false).build())
          .add(
              new ColumnSchema.ColumnSchemaBuilder("DOUBLE", Type.DOUBLE).nullable(false).build())
          .add(new ColumnSchema.ColumnSchemaBuilder("FLOAT", Type.FLOAT).nullable(true).build())
          .add(new ColumnSchema.ColumnSchemaBuilder("INT8", Type.INT8).nullable(true).build())
          .add(new ColumnSchema.ColumnSchemaBuilder("INT16", Type.INT16).nullable(true).build())
          .add(new ColumnSchema.ColumnSchemaBuilder("INT64", Type.INT64).nullable(true).build())
          .add(new ColumnSchema.ColumnSchemaBuilder("STRING", Type.STRING).nullable(true).build())
          .add(
              new ColumnSchema.ColumnSchemaBuilder("UNIXTIME_MICROS", Type.UNIXTIME_MICROS)
                  .nullable(true)
                  .build())
          .build();

  return new Schema(columns);
}
 
Example #24
Source File: KuduTableProperties.java    From presto-kudu with Apache License 2.0 6 votes vote down vote up
private static LinkedHashMap<String, ColumnDesign> getColumns(KuduTable table) {
    Schema schema = table.getSchema();
    LinkedHashMap<String, ColumnDesign> columns = new LinkedHashMap<>();
    for (ColumnSchema columnSchema : schema.getColumns()) {
        ColumnDesign design = new ColumnDesign();
        design.setNullable(columnSchema.isNullable());
        design.setKey(columnSchema.isKey());
        if (columnSchema.getCompressionAlgorithm() != null) {
            design.setCompression(columnSchema.getCompressionAlgorithm().name());
        }
        if (columnSchema.getEncoding() != null) {
            design.setEncoding(columnSchema.getEncoding().name());
        }
        columns.put(columnSchema.getName(), design);
    }
    return columns;
}
 
Example #25
Source File: KuduToBigQuery.java    From DataflowTemplates with Apache License 2.0 6 votes vote down vote up
@Override
public TableRow apply(RowResult input) {
  TableRow outputTableRow = new TableRow();
  Schema kuduSchema = input.getSchema();

  for (int i = 0; i < kuduSchema.getColumnCount(); i++) {
    String columnName = kuduSchema.getColumnByIndex(i).getName();

    if (kuduSchema.getColumnByIndex(i).getType() == Type.UNIXTIME_MICROS) {
      outputTableRow.set(columnName, convertNanoSecondsToMilliSeconds(input, i));
    } else {
      // For other datatypes return the object value
      outputTableRow.set(columnName, input.getObject(i));
    }
  }
  return outputTableRow;
}
 
Example #26
Source File: KuduTableProperties.java    From presto-kudu with Apache License 2.0 6 votes vote down vote up
private static RangeBoundValue buildRangePartitionBound(KuduTable table, byte[] rangeKey) throws Exception {
    if (rangeKey.length == 0) {
        return null;
    } else {
        Schema schema = table.getSchema();
        PartitionSchema partitionSchema = table.getPartitionSchema();
        PartitionSchema.RangeSchema rangeSchema = partitionSchema.getRangeSchema();
        List<Integer> rangeColumns = rangeSchema.getColumns();

        final int numColumns = rangeColumns.size();

        PartialRow bound = KeyEncoderAccessor.decodeRangePartitionKey(schema, partitionSchema, rangeKey);

        RangeBoundValue value = new RangeBoundValue();
        ArrayList<Object> list = new ArrayList<>();
        for (int i = 0; i < numColumns; i++) {
            Object obj = toValue(schema, bound, rangeColumns.get(i));
            list.add(obj);
        }
        value.setValues(list);
        return value;
    }
}
 
Example #27
Source File: KuduMetadata.java    From presto-kudu with Apache License 2.0 6 votes vote down vote up
@Override
public ConnectorInsertTableHandle beginInsert(ConnectorSession session, ConnectorTableHandle connectorTableHandle) {
    KuduTableHandle tableHandle = fromConnectorTableHandle(session, connectorTableHandle);

    KuduTable table = tableHandle.getTable(clientSession);
    Schema schema = table.getSchema();

    List<ColumnSchema> columns = schema.getColumns();
    List<String> columnNames = columns.stream().map(ColumnSchema::getName).collect(toImmutableList());
    List<Type> columnTypes = columns.stream()
            .map(TypeHelper::fromKuduColumn).collect(toImmutableList());

    return new KuduInsertTableHandle(
            connectorId,
            tableHandle.getSchemaTableName(),
            columnNames,
            columnTypes,
            table);
}
 
Example #28
Source File: NativeKuduClientSession.java    From presto-kudu with Apache License 2.0 6 votes vote down vote up
private void createAndFillSchemasTable() throws KuduException {
    List<String> existingSchemaNames = listSchemaNamesFromTablets();
    ColumnSchema tenantColumnSchema = new ColumnSchema.ColumnSchemaBuilder("tenant", Type.STRING)
            .key(true).build();
    ColumnSchema schemaColumnSchema = new ColumnSchema.ColumnSchemaBuilder("schema", Type.STRING)
            .key(true).build();
    Schema schema = new Schema(ImmutableList.of(tenantColumnSchema, schemaColumnSchema));
    CreateTableOptions options = new CreateTableOptions();
    options.setNumReplicas(1); // TODO config
    options.addHashPartitions(ImmutableList.of(tenantColumnSchema.getName()), 2);
    KuduTable schemasTable = client.createTable(rawSchemasTableName, schema, options);
    KuduSession session = client.newSession();
    session.setFlushMode(SessionConfiguration.FlushMode.AUTO_FLUSH_BACKGROUND);
    try {
        for (String schemaName : existingSchemaNames) {
            Insert insert = schemasTable.newInsert();
            fillSchemaRow(insert.getRow(), schemaName);
            session.apply(insert);
        }
    } finally {
        session.close();
    }
}
 
Example #29
Source File: KuduTestBase.java    From bahir-flink with Apache License 2.0 6 votes vote down vote up
protected void validateSingleKey(String tableName) throws Exception {
    KuduTable kuduTable = harness.getClient().openTable(tableName);
    Schema schema = kuduTable.getSchema();

    assertEquals(1, schema.getPrimaryKeyColumnCount());
    assertEquals(2, schema.getColumnCount());

    assertTrue(schema.getColumn("first").isKey());
    assertFalse(schema.getColumn("second").isKey());

    KuduScanner scanner = harness.getClient().newScannerBuilder(kuduTable).build();
    List<RowResult> rows = new ArrayList<>();
    scanner.forEach(rows::add);

    assertEquals(1, rows.size());
    assertEquals("f", rows.get(0).getString("first"));
    assertEquals("s", rows.get(0).getString("second"));
}
 
Example #30
Source File: KuduProducerTest.java    From syndesis with Apache License 2.0 5 votes vote down vote up
@Ignore
public void createTable() throws InterruptedException, KuduException {
    deleteTestTable(TABLE, HOST + ":" + PORT);

    errorEndpoint.expectedMessageCount(0);
    successEndpoint.expectedMessageCount(1);

    final Map<String, Object> headers = new HashMap<>();

    final List<ColumnSchema> columns = new ArrayList<>(5);
    final List<String> columnNames = Arrays.asList("id", "title", "name", "lastname", "address");

    for (int i = 0; i < columnNames.size(); i++) {
        columns.add(
            new ColumnSchema.ColumnSchemaBuilder(columnNames.get(i), Type.STRING)
                .key(i == 0)
                .build());
    }

    final List<String> rangeKeys = new ArrayList<>();
    rangeKeys.add("id");

    headers.put("Schema", new Schema(columns));
    headers.put("TableOptions", new CreateTableOptions().setRangePartitionColumns(rangeKeys));

    requestBodyAndHeaders("direct://create", null, headers);

    errorEndpoint.assertIsSatisfied();
    successEndpoint.assertIsSatisfied();
}