Java Code Examples for org.apache.kudu.client.KuduTable#getSchema()

@Override
public ConnectorInsertTableHandle beginInsert(ConnectorSession session, ConnectorTableHandle connectorTableHandle)
{
    KuduTableHandle tableHandle = (KuduTableHandle) connectorTableHandle;

    KuduTable table = tableHandle.getTable(clientSession);
    Schema schema = table.getSchema();

    List<ColumnSchema> columns = schema.getColumns();
    List<Type> columnTypes = columns.stream()
            .map(TypeHelper::fromKuduColumn).collect(toImmutableList());

    return new KuduInsertTableHandle(
            tableHandle.getSchemaTableName(),
            columnTypes,
            table);
}
 

protected KuduDataIndexRead(
    final short adapterId,
    final byte[][] dataIds,
    final KuduTable table,
    final KuduOperations operations,
    final boolean visibilityEnabled,
    final Predicate<GeoWaveRow> filter) {
  this.adapterId = adapterId;
  this.dataIds = dataIds;
  this.table = table;
  this.schema = table.getSchema();
  this.operations = operations;
  this.visibilityEnabled = visibilityEnabled;
  this.filter = filter;
  this.results = new ArrayList<>();
}
 

private void validateMultiKey(String tableName) throws Exception {
    KuduTable kuduTable = harness.getClient().openTable(tableName);
    Schema schema = kuduTable.getSchema();

    assertEquals(2, schema.getPrimaryKeyColumnCount());
    assertEquals(3, schema.getColumnCount());

    assertTrue(schema.getColumn("first").isKey());
    assertTrue(schema.getColumn("second").isKey());

    assertFalse(schema.getColumn("third").isKey());

    KuduScanner scanner = harness.getClient().newScannerBuilder(kuduTable).build();
    List<RowResult> rows = new ArrayList<>();
    scanner.forEach(rows::add);

    assertEquals(1, rows.size());
    assertEquals("f", rows.get(0).getString("first"));
    assertEquals(2, rows.get(0).getInt("second"));
    assertEquals("t", rows.get(0).getString("third"));
}
 

protected KuduRangeRead(
    final Collection<SinglePartitionQueryRanges> ranges,
    final short[] adapterIds,
    final KuduTable table,
    final KuduOperations operations,
    final boolean visibilityEnabled,
    final Predicate<GeoWaveRow> filter,
    final GeoWaveRowIteratorTransformer<T> rowTransformer,
    final boolean rowMerging) {
  this.ranges = ranges;
  this.adapterIds = adapterIds;
  this.table = table;
  this.schema = table.getSchema();
  this.operations = operations;
  this.visibilityEnabled = visibilityEnabled;
  this.filter = filter;
  this.rowTransformer = rowTransformer;
  this.rowMerging = rowMerging;
}
 

protected void validateSingleKey(String tableName) throws Exception {
    KuduTable kuduTable = harness.getClient().openTable(tableName);
    Schema schema = kuduTable.getSchema();

    assertEquals(1, schema.getPrimaryKeyColumnCount());
    assertEquals(2, schema.getColumnCount());

    assertTrue(schema.getColumn("first").isKey());
    assertFalse(schema.getColumn("second").isKey());

    KuduScanner scanner = harness.getClient().newScannerBuilder(kuduTable).build();
    List<RowResult> rows = new ArrayList<>();
    scanner.forEach(rows::add);

    assertEquals(1, rows.size());
    assertEquals("f", rows.get(0).getString("first"));
    assertEquals("s", rows.get(0).getString("second"));
}
 

private static LinkedHashMap<String, ColumnDesign> getColumns(KuduTable table) {
    Schema schema = table.getSchema();
    LinkedHashMap<String, ColumnDesign> columns = new LinkedHashMap<>();
    for (ColumnSchema columnSchema : schema.getColumns()) {
        ColumnDesign design = new ColumnDesign();
        design.setNullable(columnSchema.isNullable());
        design.setKey(columnSchema.isKey());
        if (columnSchema.getCompressionAlgorithm() != null) {
            design.setCompression(columnSchema.getCompressionAlgorithm().name());
        }
        if (columnSchema.getEncoding() != null) {
            design.setEncoding(columnSchema.getEncoding().name());
        }
        columns.put(columnSchema.getName(), design);
    }
    return columns;
}
 

private static RangeBoundValue buildRangePartitionBound(KuduTable table, byte[] rangeKey) throws Exception {
    if (rangeKey.length == 0) {
        return null;
    } else {
        Schema schema = table.getSchema();
        PartitionSchema partitionSchema = table.getPartitionSchema();
        PartitionSchema.RangeSchema rangeSchema = partitionSchema.getRangeSchema();
        List<Integer> rangeColumns = rangeSchema.getColumns();

        final int numColumns = rangeColumns.size();

        PartialRow bound = KeyEncoderAccessor.decodeRangePartitionKey(schema, partitionSchema, rangeKey);

        RangeBoundValue value = new RangeBoundValue();
        ArrayList<Object> list = new ArrayList<>();
        for (int i = 0; i < numColumns; i++) {
            Object obj = toValue(schema, bound, rangeColumns.get(i));
            list.add(obj);
        }
        value.setValues(list);
        return value;
    }
}
 

@Override
public ConnectorInsertTableHandle beginInsert(ConnectorSession session, ConnectorTableHandle connectorTableHandle) {
    KuduTableHandle tableHandle = fromConnectorTableHandle(session, connectorTableHandle);

    KuduTable table = tableHandle.getTable(clientSession);
    Schema schema = table.getSchema();

    List<ColumnSchema> columns = schema.getColumns();
    List<String> columnNames = columns.stream().map(ColumnSchema::getName).collect(toImmutableList());
    List<Type> columnTypes = columns.stream()
            .map(TypeHelper::fromKuduColumn).collect(toImmutableList());

    return new KuduInsertTableHandle(
            connectorId,
            tableHandle.getSchemaTableName(),
            columnNames,
            columnTypes,
            table);
}
 

private static RangeBoundValue buildRangePartitionBound(KuduTable table, byte[] rangeKey)
{
    if (rangeKey.length == 0) {
        return null;
    }
    else {
        Schema schema = table.getSchema();
        PartitionSchema partitionSchema = table.getPartitionSchema();
        PartitionSchema.RangeSchema rangeSchema = partitionSchema.getRangeSchema();
        List<Integer> rangeColumns = rangeSchema.getColumns();

        final int numColumns = rangeColumns.size();

        PartialRow bound = KeyEncoderAccessor.decodeRangePartitionKey(schema, partitionSchema, rangeKey);

        ArrayList<Object> list = new ArrayList<>();
        for (int i = 0; i < numColumns; i++) {
            Object obj = toValue(schema, bound, rangeColumns.get(i));
            list.add(obj);
        }
        return new RangeBoundValue(list);
    }
}
 

public static MetaTable getMetaTable(List<String> masterAddresses, String tableName, String[] columns) throws Exception {
    KuduClient client = null;
    try {
        client = createClient(masterAddresses);
        KuduTable kuduTable = client.openTable(tableName);
        Schema schema = kuduTable.getSchema();
        List<ColumnSchema> columnSchemas = schema.getColumns();
        Set<String> realColumnNames = new HashSet<String>();
        for (ColumnSchema cs : columnSchemas) {
            realColumnNames.add(cs.getName());
        }
        Set<String> inputColumnNames = new HashSet<String>(Arrays.asList(columns));
        LOG.info("config column:"  + ArrayUtils.toString(inputColumnNames.toString()));
        LOG.info("kudu column:"  + ArrayUtils.toString(realColumnNames));
        inputColumnNames.removeAll(realColumnNames);

        //如果kudu不包含输入字段,抛出异常
        if (inputColumnNames.size() != 0) {
        	ErrorRecord.addError(String.format("字段配置与kudu实际不相符{%s}!",ArrayUtils.toString(inputColumnNames.toString())));
            throw new Exception(String.format("字段配置与kudu实际不相符{%s}!",ArrayUtils.toString(inputColumnNames.toString())));
        }

        return new MetaTable(columnSchemas);
    } catch (KuduException e) {
        throw e;
    } finally {
        closeClient(client);
    }
}
 

private ConnectorTableMetadata getTableMetadata(KuduTableHandle tableHandle) {
    KuduTable table = tableHandle.getTable(clientSession);
    Schema schema = table.getSchema();

    List<ColumnMetadata> columnsMetaList = schema.getColumns().stream()
            .filter(col -> !col.isKey() || !col.getName().equals(KuduColumnHandle.ROW_ID))
            .map(col -> {
                StringBuilder extra = new StringBuilder();
                if (col.isKey()) {
                    extra.append("key, ");
                } else if (col.isNullable()) {
                    extra.append("nullable, ");
                }
                if (col.getEncoding() != null) {
                    extra.append("encoding=").append(col.getEncoding().name()).append(", ");
                }
                if (col.getCompressionAlgorithm() != null) {
                    extra.append("compression=").append(col.getCompressionAlgorithm().name()).append(", ");
                }
                if (extra.length() > 2) {
                    extra.setLength(extra.length() - 2);
                }
                Type prestoType = TypeHelper.fromKuduColumn(col);
                return new ColumnMetadata(col.getName(), prestoType, null, extra.toString(), false);
            }).collect(toImmutableList());

    Map<String, Object> properties = clientSession.getTableProperties(tableHandle);
    return new ConnectorTableMetadata(tableHandle.getSchemaTableName(), columnsMetaList, properties);
}
 

private void changeRangePartition(SchemaTableName schemaTableName, RangePartition rangePartition,
        RangePartitionChange change)
{
    try {
        String rawName = schemaEmulation.toRawName(schemaTableName);
        KuduTable table = client.openTable(rawName);
        Schema schema = table.getSchema();
        PartitionDesign design = KuduTableProperties.getPartitionDesign(table);
        RangePartitionDefinition definition = design.getRange();
        if (definition == null) {
            throw new PrestoException(QUERY_REJECTED, "Table " + schemaTableName + " has no range partition");
        }
        PartialRow lowerBound = KuduTableProperties.toRangeBoundToPartialRow(schema, definition, rangePartition.getLower());
        PartialRow upperBound = KuduTableProperties.toRangeBoundToPartialRow(schema, definition, rangePartition.getUpper());
        AlterTableOptions alterOptions = new AlterTableOptions();
        switch (change) {
            case ADD:
                alterOptions.addRangePartition(lowerBound, upperBound);
                break;
            case DROP:
                alterOptions.dropRangePartition(lowerBound, upperBound);
                break;
        }
        client.alterTable(rawName, alterOptions);
    }
    catch (KuduException e) {
        throw new PrestoException(GENERIC_INTERNAL_ERROR, e);
    }
}
 

public static PartitionDesign getPartitionDesign(KuduTable table)
{
    Schema schema = table.getSchema();
    PartitionDesign partitionDesign = new PartitionDesign();
    PartitionSchema partitionSchema = table.getPartitionSchema();

    List<HashPartitionDefinition> hashPartitions = partitionSchema.getHashBucketSchemas().stream()
            .map(hashBucketSchema -> {
                HashPartitionDefinition hash = new HashPartitionDefinition();
                List<String> cols = hashBucketSchema.getColumnIds().stream()
                        .map(idx -> schema.getColumnByIndex(idx).getName()).collect(toImmutableList());
                hash.setColumns(cols);
                hash.setBuckets(hashBucketSchema.getNumBuckets());
                return hash;
            }).collect(toImmutableList());
    partitionDesign.setHash(hashPartitions);

    List<Integer> rangeColumns = partitionSchema.getRangeSchema().getColumns();
    if (!rangeColumns.isEmpty()) {
        RangePartitionDefinition definition = new RangePartitionDefinition();
        definition.setColumns(rangeColumns.stream()
                .map(i -> schema.getColumns().get(i).getName())
                .collect(ImmutableList.toImmutableList()));
        partitionDesign.setRange(definition);
    }

    return partitionDesign;
}
 

public static PartitionDesign getPartitionDesign(KuduTable table) {
    Schema schema = table.getSchema();
    PartitionDesign partitionDesign = new PartitionDesign();
    PartitionSchema partitionSchema = table.getPartitionSchema();

    List<HashPartitionDefinition> hashPartitions = partitionSchema.getHashBucketSchemas().stream()
            .map(hashBucketSchema -> {
                HashPartitionDefinition hash = new HashPartitionDefinition();
                List<String> cols = hashBucketSchema.getColumnIds().stream()
                        .map(idx -> schema.getColumnByIndex(idx).getName()).collect(toImmutableList());
                hash.setColumns(cols);
                hash.setBuckets(hashBucketSchema.getNumBuckets());
                return hash;
            }).collect(toImmutableList());
    partitionDesign.setHash(hashPartitions);

    List<Integer> rangeColumns = partitionSchema.getRangeSchema().getColumns();
    if (!rangeColumns.isEmpty()) {
        RangePartitionDefinition definition = new RangePartitionDefinition();
        definition.setColumns(rangeColumns.stream()
                .map(i -> schema.getColumns().get(i).getName())
                .collect(ImmutableList.toImmutableList()));
        partitionDesign.setRange(definition);
    }

    return partitionDesign;
}
 

private static LinkedHashMap<String, ColumnDesign> getColumns(KuduTable table)
{
    Schema schema = table.getSchema();
    LinkedHashMap<String, ColumnDesign> columns = new LinkedHashMap<>();
    for (ColumnSchema columnSchema : schema.getColumns()) {
        ColumnDesign design = new ColumnDesign();
        design.setNullable(columnSchema.isNullable());
        design.setPrimaryKey(columnSchema.isKey());
        design.setCompression(lookupCompressionString(columnSchema.getCompressionAlgorithm()));
        design.setEncoding(lookupEncodingString(columnSchema.getEncoding()));
        columns.put(columnSchema.getName(), design);
    }
    return columns;
}
 

private ConnectorTableMetadata getTableMetadata(KuduTableHandle tableHandle)
{
    KuduTable table = tableHandle.getTable(clientSession);
    Schema schema = table.getSchema();

    List<ColumnMetadata> columnsMetaList = schema.getColumns().stream()
            .filter(column -> !column.isKey() || !column.getName().equals(KuduColumnHandle.ROW_ID))
            .map(this::getColumnMetadata)
            .collect(toImmutableList());

    Map<String, Object> properties = clientSession.getTableProperties(tableHandle);
    return new ConnectorTableMetadata(tableHandle.getSchemaTableName(), columnsMetaList, properties);
}
 

private void insert(KuduTable kuduTable, Operation operation, Record record, List<String> fieldNames){
    PartialRow row = operation.getRow();
    Schema colSchema = kuduTable.getSchema();

    for (String colName : fieldNames) {
        int colIdx = this.getColumnIndex(colSchema, colName);
        if (colIdx != -1) {
            Type colType = colSchema.getColumnByIndex(colIdx).getType();

            switch (colType.getDataType()) {
                case BOOL:
                    row.addBoolean(colIdx, record.getAsBoolean(colName));
                    break;
                case FLOAT:
                    row.addFloat(colIdx, record.getAsFloat(colName));
                    break;
                case DOUBLE:
                    row.addDouble(colIdx, record.getAsDouble(colName));
                    break;
                case BINARY:
                    row.addBinary(colIdx, record.getAsString(colName).getBytes());
                    break;
                case INT8:
                case INT16:
                    short temp = (short)record.getAsInt(colName).intValue();
                    row.addShort(colIdx, temp);
                case INT32:
                    row.addInt(colIdx, record.getAsInt(colName));
                    break;
                case INT64:
                    row.addLong(colIdx, record.getAsLong(colName));
                    break;
                case STRING:
                    row.addString(colIdx, record.getAsString(colName));
                    break;
                default:
                    throw new IllegalStateException(String.format("unknown column type %s", colType));
            }
        }
    }
}
 

@Override
public Schema getTableSchema(KuduTableHandle tableHandle) {
    KuduTable table = tableHandle.getTable(this);
    return table.getSchema();
}
 

public Schema getTableSchema(KuduTableHandle tableHandle)
{
    KuduTable table = tableHandle.getTable(this);
    return table.getSchema();
}
 

@Override
public ConnectorOutputTableHandle beginCreateTable(
        ConnectorSession session,
        ConnectorTableMetadata tableMetadata,
        Optional<ConnectorNewTableLayout> layout)
{
    PartitionDesign design = KuduTableProperties.getPartitionDesign(tableMetadata.getProperties());
    boolean generateUUID = !design.hasPartitions();
    ConnectorTableMetadata finalTableMetadata = tableMetadata;
    if (generateUUID) {
        String rowId = KuduColumnHandle.ROW_ID;
        List<ColumnMetadata> copy = new ArrayList<>(tableMetadata.getColumns());
        Map<String, Object> columnProperties = new HashMap<>();
        columnProperties.put(KuduTableProperties.PRIMARY_KEY, true);
        copy.add(0, ColumnMetadata.builder()
                .setName(rowId)
                .setType(VarcharType.VARCHAR)
                .setComment(Optional.of("key=true"))
                .setHidden(true)
                .setProperties(columnProperties)
                .build());
        List<ColumnMetadata> finalColumns = ImmutableList.copyOf(copy);
        Map<String, Object> propsCopy = new HashMap<>(tableMetadata.getProperties());
        propsCopy.put(KuduTableProperties.PARTITION_BY_HASH_COLUMNS, ImmutableList.of(rowId));
        propsCopy.put(KuduTableProperties.PARTITION_BY_HASH_BUCKETS, 2);
        Map<String, Object> finalProperties = ImmutableMap.copyOf(propsCopy);
        finalTableMetadata = new ConnectorTableMetadata(tableMetadata.getTable(),
                finalColumns, finalProperties, tableMetadata.getComment());
    }
    KuduTable table = clientSession.createTable(finalTableMetadata, false);

    Schema schema = table.getSchema();

    List<ColumnSchema> columns = schema.getColumns();
    List<Type> columnTypes = columns.stream()
            .map(TypeHelper::fromKuduColumn).collect(toImmutableList());
    List<Type> columnOriginalTypes = finalTableMetadata.getColumns().stream()
            .map(ColumnMetadata::getType).collect(toImmutableList());

    return new KuduOutputTableHandle(
            finalTableMetadata.getTable(),
            columnOriginalTypes,
            columnTypes,
            generateUUID,
            table);
}