org.apache.kudu.ColumnTypeAttributes Java Examples

The following examples show how to use org.apache.kudu.ColumnTypeAttributes. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: KuduTableUtils.java    From bahir-flink with Apache License 2.0 6 votes vote down vote up 
public static List<ColumnSchema> toKuduConnectorColumns(List<Tuple2<String, DataType>> columns, Collection<String> keyColumns) {
    return columns.stream()
            .map(t -> {
                        ColumnSchema.ColumnSchemaBuilder builder = new ColumnSchema
                                .ColumnSchemaBuilder(t.f0, KuduTypeUtils.toKuduType(t.f1))
                                .key(keyColumns.contains(t.f0))
                                .nullable(!keyColumns.contains(t.f0) && t.f1.getLogicalType().isNullable());
                        if(t.f1.getLogicalType() instanceof DecimalType) {
                            DecimalType decimalType = ((DecimalType) t.f1.getLogicalType());
                            builder.typeAttributes(new ColumnTypeAttributes.ColumnTypeAttributesBuilder()
                                .precision(decimalType.getPrecision())
                                .scale(decimalType.getScale())
                                .build());
                        }
                        return builder.build();
                    }
            ).collect(Collectors.toList());
}
Example #2
Source File: KuduClientSession.java    From presto with Apache License 2.0 5 votes vote down vote up 
private void setTypeAttributes(ColumnMetadata columnMetadata, ColumnSchema.ColumnSchemaBuilder builder)
{
    if (columnMetadata.getType() instanceof DecimalType) {
        DecimalType type = (DecimalType) columnMetadata.getType();
        ColumnTypeAttributes attributes = new ColumnTypeAttributes.ColumnTypeAttributesBuilder()
                .precision(type.getPrecision())
                .scale(type.getScale()).build();
        builder.typeAttributes(attributes);
    }
}
Example #3
Source File: TypeHelper.java    From presto with Apache License 2.0 5 votes vote down vote up 
private static Type fromKuduClientType(org.apache.kudu.Type ktype, ColumnTypeAttributes attributes)
{
    switch (ktype) {
        case STRING:
            return VarcharType.VARCHAR;
        case UNIXTIME_MICROS:
            return TimestampType.TIMESTAMP;
        case INT64:
            return BigintType.BIGINT;
        case INT32:
            return IntegerType.INTEGER;
        case INT16:
            return SmallintType.SMALLINT;
        case INT8:
            return TinyintType.TINYINT;
        case FLOAT:
            return RealType.REAL;
        case DOUBLE:
            return DoubleType.DOUBLE;
        case BOOL:
            return BooleanType.BOOLEAN;
        case BINARY:
            return VarbinaryType.VARBINARY;
        case DECIMAL:
            return DecimalType.createDecimalType(attributes.getPrecision(), attributes.getScale());
        default:
            throw new IllegalStateException("Kudu type not implemented for " + ktype);
    }
}
Example #4
Source File: NativeKuduClientSession.java    From presto-kudu with Apache License 2.0 5 votes vote down vote up 
private void setTypeAttributes(ColumnMetadata columnMetadata, ColumnSchema.ColumnSchemaBuilder builder) {
    if (columnMetadata.getType() instanceof DecimalType) {
        DecimalType type = (DecimalType) columnMetadata.getType();
        ColumnTypeAttributes attributes = new ColumnTypeAttributes.ColumnTypeAttributesBuilder()
                .precision(type.getPrecision())
                .scale(type.getScale()).build();
        builder.typeAttributes(attributes);
    }
}
Example #5
Source File: TypeHelper.java    From presto-kudu with Apache License 2.0 5 votes vote down vote up 
private static Type fromKuduClientType(org.apache.kudu.Type ktype, ColumnTypeAttributes attributes) {
    switch (ktype) {
        case STRING:
            return VarcharType.VARCHAR;
        case UNIXTIME_MICROS:
            return TimestampType.TIMESTAMP;
        case INT64:
            return BigintType.BIGINT;
        case INT32:
            return IntegerType.INTEGER;
        case INT16:
            return SmallintType.SMALLINT;
        case INT8:
            return TinyintType.TINYINT;
        case FLOAT:
            return RealType.REAL;
        case DOUBLE:
            return DoubleType.DOUBLE;
        case BOOL:
            return BooleanType.BOOLEAN;
        case BINARY:
            return VarbinaryType.VARBINARY;
        case DECIMAL:
            return DecimalType.createDecimalType(attributes.getPrecision(), attributes.getScale());
        default:
            throw new IllegalStateException("Kudu type not implemented for " + ktype);
    }
}
Example #6
Source File: KuduTypeUtils.java    From bahir-flink with Apache License 2.0 5 votes vote down vote up 
public static DataType toFlinkType(Type type, ColumnTypeAttributes typeAttributes) {
    switch (type) {
        case STRING:
            return DataTypes.STRING();
        case FLOAT:
            return DataTypes.FLOAT();
        case INT8:
            return DataTypes.TINYINT();
        case INT16:
            return DataTypes.SMALLINT();
        case INT32:
            return DataTypes.INT();
        case INT64:
            return DataTypes.BIGINT();
        case DOUBLE:
            return DataTypes.DOUBLE();
        case DECIMAL:
            return DataTypes.DECIMAL(typeAttributes.getPrecision(), typeAttributes.getScale());
        case BOOL:
            return DataTypes.BOOLEAN();
        case BINARY:
            return DataTypes.BYTES();
        case UNIXTIME_MICROS:
            return new AtomicDataType(new TimestampType(3), Timestamp.class);

        default:
            throw new IllegalArgumentException("Illegal var type: " + type);
    }
}
Example #7
Source File: AbstractKuduProcessor.java    From nifi with Apache License 2.0 5 votes vote down vote up 
private ColumnTypeAttributes getKuduTypeAttributes(final DataType nifiType) {
    if (nifiType.getFieldType().equals(RecordFieldType.DECIMAL)) {
        final DecimalDataType decimalDataType = (DecimalDataType) nifiType;
        return new ColumnTypeAttributes.ColumnTypeAttributesBuilder().precision(decimalDataType.getPrecision()).scale(decimalDataType.getScale()).build();
    } else {
        return null;
    }
}
Example #8
Source File: ITPutKudu.java    From nifi with Apache License 2.0 5 votes vote down vote up 
private void createKuduTable() throws KuduException {
    KuduClient client =  harness.getClient();
    List<ColumnSchema> columns = new ArrayList<>();
    columns.add(new ColumnSchema.ColumnSchemaBuilder("id", Type.INT32).key(true).build());
    columns.add(new ColumnSchema.ColumnSchemaBuilder("stringval", Type.STRING).build());
    columns.add(new ColumnSchema.ColumnSchemaBuilder("varcharval", Type.VARCHAR).typeAttributes(
            new ColumnTypeAttributes.ColumnTypeAttributesBuilder().length(256).build()
    ).build());
    columns.add(new ColumnSchema.ColumnSchemaBuilder("num32val", Type.INT32).build());
    columns.add(new ColumnSchema.ColumnSchemaBuilder("timestampval", Type.UNIXTIME_MICROS).build());
    Schema schema = new Schema(columns);
    CreateTableOptions opts = new CreateTableOptions()
        .addHashPartitions(Collections.singletonList("id"), 4);
    client.createTable(DEFAULT_TABLE_NAME, schema, opts);
}
Example #9
Source File: TestPutKudu.java    From nifi with Apache License 2.0 5 votes vote down vote up 
private PartialRow buildPartialRow(Long id, String name, Short age, String kuduIdName, String recordIdName, String airport_code, Boolean lowercaseFields) {
    final Schema kuduSchema = new Schema(Arrays.asList(
        new ColumnSchema.ColumnSchemaBuilder(kuduIdName, Type.INT64).key(true).build(),
        new ColumnSchema.ColumnSchemaBuilder("name", Type.STRING).nullable(true).build(),
        new ColumnSchema.ColumnSchemaBuilder("age", Type.INT16).nullable(false).build(),
        new ColumnSchema.ColumnSchemaBuilder("updated_at", Type.UNIXTIME_MICROS).nullable(false).build(),
        new ColumnSchema.ColumnSchemaBuilder("score", Type.DECIMAL).nullable(true).typeAttributes(
            new ColumnTypeAttributes.ColumnTypeAttributesBuilder().precision(9).scale(0).build()
        ).build(),
        new ColumnSchema.ColumnSchemaBuilder("airport_code", Type.VARCHAR).nullable(true).typeAttributes(
                new ColumnTypeAttributes.ColumnTypeAttributesBuilder().length(3).build()
        ).build()));

    final RecordSchema schema = new SimpleRecordSchema(Arrays.asList(
        new RecordField(recordIdName, RecordFieldType.BIGINT.getDataType()),
        new RecordField("name", RecordFieldType.STRING.getDataType()),
        new RecordField("age", RecordFieldType.SHORT.getDataType()),
        new RecordField("updated_at", RecordFieldType.TIMESTAMP.getDataType()),
        new RecordField("score", RecordFieldType.LONG.getDataType()),
        new RecordField("airport_code", RecordFieldType.STRING.getDataType())));

    Map<String, Object> values = new HashMap<>();
    PartialRow row = kuduSchema.newPartialRow();
    values.put(recordIdName, id);
    values.put("name", name);
    values.put("age", age);
    values.put("updated_at", new Timestamp(System.currentTimeMillis()));
    values.put("score", 10000L);
    values.put("airport_code", airport_code);
    processor.buildPartialRow(
        kuduSchema,
        row,
        new MapRecord(schema, values),
        schema.getFieldNames(),
            true,
        lowercaseFields
    );
    return row;
}
Example #10
Source File: KuduLookupService.java    From nifi with Apache License 2.0 4 votes vote down vote up 
private RecordSchema kuduSchemaToNiFiSchema(Schema kuduTableSchema, List<String> columnNames){
    final List<RecordField> fields = new ArrayList<>();
    for(String columnName : columnNames) {
        if(!kuduTableSchema.hasColumn(columnName)){
            throw new IllegalArgumentException("Column not found in Kudu table schema " + columnName);
        }
        ColumnSchema cs = kuduTableSchema.getColumn(columnName);
        switch (cs.getType()) {
            case INT8:
                fields.add(new RecordField(cs.getName(), RecordFieldType.BYTE.getDataType()));
                break;
            case INT16:
                fields.add(new RecordField(cs.getName(), RecordFieldType.SHORT.getDataType()));
                break;
            case INT32:
                fields.add(new RecordField(cs.getName(), RecordFieldType.INT.getDataType()));
                break;
            case INT64:
                fields.add(new RecordField(cs.getName(), RecordFieldType.LONG.getDataType()));
                break;
            case DECIMAL:
                final ColumnTypeAttributes attributes = cs.getTypeAttributes();
                fields.add(new RecordField(cs.getName(), RecordFieldType.DECIMAL.getDecimalDataType(attributes.getPrecision(), attributes.getScale())));
                break;
            case UNIXTIME_MICROS:
                fields.add(new RecordField(cs.getName(), RecordFieldType.TIMESTAMP.getDataType()));
                break;
            case BINARY:
            case STRING:
            case VARCHAR:
                fields.add(new RecordField(cs.getName(), RecordFieldType.STRING.getDataType()));
                break;
            case DOUBLE:
                fields.add(new RecordField(cs.getName(), RecordFieldType.DOUBLE.getDataType()));
                break;
            case BOOL:
                fields.add(new RecordField(cs.getName(), RecordFieldType.BOOLEAN.getDataType()));
                break;
            case FLOAT:
                fields.add(new RecordField(cs.getName(), RecordFieldType.FLOAT.getDataType()));
                break;
        }
    }
    return new SimpleRecordSchema(fields);
}
Example #11
Source File: ITestKuduLookupService.java    From nifi with Apache License 2.0 4 votes vote down vote up 
@Before
public void init() throws Exception {
    testRunner = TestRunners.newTestRunner(SampleProcessor.class);
    testRunner.setValidateExpressionUsage(false);
    final String tableName = "table1";

    KuduClient client =  harness.getClient();
    List<ColumnSchema> columns = new ArrayList<>();
    columns.add(new ColumnSchema.ColumnSchemaBuilder("string", Type.STRING).key(true).build());
    columns.add(new ColumnSchema.ColumnSchemaBuilder("binary", Type.BINARY).build());
    columns.add(new ColumnSchema.ColumnSchemaBuilder("bool", Type.BOOL).build());
    columns.add(new ColumnSchema
            .ColumnSchemaBuilder("decimal", Type.DECIMAL)
            .typeAttributes(DecimalUtil.typeAttributes(DecimalUtil.MAX_DECIMAL64_PRECISION, 1))
            .build()
    );
    columns.add(new ColumnSchema.ColumnSchemaBuilder("double", Type.DOUBLE).build());
    columns.add(new ColumnSchema.ColumnSchemaBuilder("float", Type.FLOAT).build());
    columns.add(new ColumnSchema.ColumnSchemaBuilder("int8", Type.INT8).build());
    columns.add(new ColumnSchema.ColumnSchemaBuilder("int16", Type.INT16).build());
    columns.add(new ColumnSchema.ColumnSchemaBuilder("int32", Type.INT32).build());
    columns.add(new ColumnSchema.ColumnSchemaBuilder("int64", Type.INT64).build());
    columns.add(new ColumnSchema.ColumnSchemaBuilder("unixtime_micros", Type.UNIXTIME_MICROS).build());
    columns.add(new ColumnSchema.ColumnSchemaBuilder("varchar_3", Type.VARCHAR).typeAttributes(
            new ColumnTypeAttributes.ColumnTypeAttributesBuilder().length(3).build()
    ).build());
    Schema schema = new Schema(columns);

    CreateTableOptions opts = new CreateTableOptions().setRangePartitionColumns(Collections.singletonList("string"));
    client.createTable(tableName, schema, opts);

    KuduTable table = client.openTable(tableName);
    KuduSession session = client.newSession();

    Insert insert = table.newInsert();
    PartialRow row = insert.getRow();
    row.addString("string", "string1");
    row.addBinary("binary", "binary1".getBytes());
    row.addBoolean("bool",true);
    row.addDecimal("decimal", BigDecimal.valueOf(0.1));
    row.addDouble("double",0.2);
    row.addFloat("float",0.3f);
    row.addByte("int8", (byte) 1);
    row.addShort("int16", (short) 2);
    row.addInt("int32",3);
    row.addLong("int64",4L);
    row.addTimestamp("unixtime_micros", new Timestamp(nowMillis));
    row.addVarchar("varchar_3", "SFO");
    session.apply(insert);

    insert = table.newInsert();
    row = insert.getRow();
    row.addString("string", "string2");
    row.addBinary("binary", "binary2".getBytes());
    row.addBoolean("bool",false);
    row.addDecimal("decimal", BigDecimal.valueOf(0.1));
    row.addDouble("double",1.2);
    row.addFloat("float",1.3f);
    row.addByte("int8", (byte) 11);
    row.addShort("int16", (short) 12);
    row.addInt("int32",13);
    row.addLong("int64",14L);
    row.addTimestamp("unixtime_micros", new Timestamp(nowMillis+(1000L * 60 * 60 * 24 * 365))); //+ 1 year
    row.addVarchar("varchar_3", "SJC");
    session.apply(insert);

    session.close();

    kuduLookupService = new KuduLookupService();
    testRunner.addControllerService("kuduLookupService", kuduLookupService);
    testRunner.setProperty(kuduLookupService, KuduLookupService.KUDU_MASTERS, "testLocalHost:7051");
    testRunner.setProperty(kuduLookupService, KuduLookupService.KUDU_REPLICA_SELECTION, KuduLookupService.LEADER_ONLY);
    testRunner.setProperty(kuduLookupService, KuduLookupService.TABLE_NAME, tableName);
    kuduLookupService.kuduClient = client;
}