Java Code Examples for org.apache.kudu.Schema#getColumns()

@Override
public ConnectorInsertTableHandle beginInsert(ConnectorSession session, ConnectorTableHandle connectorTableHandle)
{
    KuduTableHandle tableHandle = (KuduTableHandle) connectorTableHandle;

    KuduTable table = tableHandle.getTable(clientSession);
    Schema schema = table.getSchema();

    List<ColumnSchema> columns = schema.getColumns();
    List<Type> columnTypes = columns.stream()
            .map(TypeHelper::fromKuduColumn).collect(toImmutableList());

    return new KuduInsertTableHandle(
            tableHandle.getSchemaTableName(),
            columnTypes,
            table);
}
 

@Override
public ConnectorInsertTableHandle beginInsert(ConnectorSession session, ConnectorTableHandle connectorTableHandle) {
    KuduTableHandle tableHandle = fromConnectorTableHandle(session, connectorTableHandle);

    KuduTable table = tableHandle.getTable(clientSession);
    Schema schema = table.getSchema();

    List<ColumnSchema> columns = schema.getColumns();
    List<String> columnNames = columns.stream().map(ColumnSchema::getName).collect(toImmutableList());
    List<Type> columnTypes = columns.stream()
            .map(TypeHelper::fromKuduColumn).collect(toImmutableList());

    return new KuduInsertTableHandle(
            connectorId,
            tableHandle.getSchemaTableName(),
            columnNames,
            columnTypes,
            table);
}
 

private static LinkedHashMap<String, ColumnDesign> getColumns(KuduTable table) {
    Schema schema = table.getSchema();
    LinkedHashMap<String, ColumnDesign> columns = new LinkedHashMap<>();
    for (ColumnSchema columnSchema : schema.getColumns()) {
        ColumnDesign design = new ColumnDesign();
        design.setNullable(columnSchema.isNullable());
        design.setKey(columnSchema.isKey());
        if (columnSchema.getCompressionAlgorithm() != null) {
            design.setCompression(columnSchema.getCompressionAlgorithm().name());
        }
        if (columnSchema.getEncoding() != null) {
            design.setEncoding(columnSchema.getEncoding().name());
        }
        columns.put(columnSchema.getName(), design);
    }
    return columns;
}
 

private static LinkedHashMap<String, ColumnDesign> getColumns(KuduTable table)
{
    Schema schema = table.getSchema();
    LinkedHashMap<String, ColumnDesign> columns = new LinkedHashMap<>();
    for (ColumnSchema columnSchema : schema.getColumns()) {
        ColumnDesign design = new ColumnDesign();
        design.setNullable(columnSchema.isNullable());
        design.setPrimaryKey(columnSchema.isKey());
        design.setCompression(lookupCompressionString(columnSchema.getCompressionAlgorithm()));
        design.setEncoding(lookupEncodingString(columnSchema.getEncoding()));
        columns.put(columnSchema.getName(), design);
    }
    return columns;
}
 

public static MetaTable getMetaTable(List<String> masterAddresses, String tableName, String[] columns) throws Exception {
    KuduClient client = null;
    try {
        client = createClient(masterAddresses);
        KuduTable kuduTable = client.openTable(tableName);
        Schema schema = kuduTable.getSchema();
        List<ColumnSchema> columnSchemas = schema.getColumns();
        Set<String> realColumnNames = new HashSet<String>();
        for (ColumnSchema cs : columnSchemas) {
            realColumnNames.add(cs.getName());
        }
        Set<String> inputColumnNames = new HashSet<String>(Arrays.asList(columns));
        LOG.info("config column:"  + ArrayUtils.toString(inputColumnNames.toString()));
        LOG.info("kudu column:"  + ArrayUtils.toString(realColumnNames));
        inputColumnNames.removeAll(realColumnNames);

        //如果kudu不包含输入字段,抛出异常
        if (inputColumnNames.size() != 0) {
        	ErrorRecord.addError(String.format("字段配置与kudu实际不相符{%s}!",ArrayUtils.toString(inputColumnNames.toString())));
            throw new Exception(String.format("字段配置与kudu实际不相符{%s}!",ArrayUtils.toString(inputColumnNames.toString())));
        }

        return new MetaTable(columnSchemas);
    } catch (KuduException e) {
        throw e;
    } finally {
        closeClient(client);
    }
}
 

public static TableSchema kuduToFlinkSchema(Schema schema) {
    TableSchema.Builder builder = TableSchema.builder();

    for (ColumnSchema column : schema.getColumns()) {
        DataType flinkType = KuduTypeUtils.toFlinkType(column.getType(), column.getTypeAttributes()).nullable();
        builder.field(column.getName(), flinkType);
    }

    return builder.build();
}
 

@Override
public ConnectorOutputTableHandle beginCreateTable(
        ConnectorSession session,
        ConnectorTableMetadata tableMetadata,
        Optional<ConnectorNewTableLayout> layout)
{
    PartitionDesign design = KuduTableProperties.getPartitionDesign(tableMetadata.getProperties());
    boolean generateUUID = !design.hasPartitions();
    ConnectorTableMetadata finalTableMetadata = tableMetadata;
    if (generateUUID) {
        String rowId = KuduColumnHandle.ROW_ID;
        List<ColumnMetadata> copy = new ArrayList<>(tableMetadata.getColumns());
        Map<String, Object> columnProperties = new HashMap<>();
        columnProperties.put(KuduTableProperties.PRIMARY_KEY, true);
        copy.add(0, ColumnMetadata.builder()
                .setName(rowId)
                .setType(VarcharType.VARCHAR)
                .setComment(Optional.of("key=true"))
                .setHidden(true)
                .setProperties(columnProperties)
                .build());
        List<ColumnMetadata> finalColumns = ImmutableList.copyOf(copy);
        Map<String, Object> propsCopy = new HashMap<>(tableMetadata.getProperties());
        propsCopy.put(KuduTableProperties.PARTITION_BY_HASH_COLUMNS, ImmutableList.of(rowId));
        propsCopy.put(KuduTableProperties.PARTITION_BY_HASH_BUCKETS, 2);
        Map<String, Object> finalProperties = ImmutableMap.copyOf(propsCopy);
        finalTableMetadata = new ConnectorTableMetadata(tableMetadata.getTable(),
                finalColumns, finalProperties, tableMetadata.getComment());
    }
    KuduTable table = clientSession.createTable(finalTableMetadata, false);

    Schema schema = table.getSchema();

    List<ColumnSchema> columns = schema.getColumns();
    List<Type> columnTypes = columns.stream()
            .map(TypeHelper::fromKuduColumn).collect(toImmutableList());
    List<Type> columnOriginalTypes = finalTableMetadata.getColumns().stream()
            .map(ColumnMetadata::getType).collect(toImmutableList());

    return new KuduOutputTableHandle(
            finalTableMetadata.getTable(),
            columnOriginalTypes,
            columnTypes,
            generateUUID,
            table);
}