Java Code Examples for org.apache.flink.table.types.logical.RowType#RowField

The following examples show how to use org.apache.flink.table.types.logical.RowType#RowField . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: LogicalTypeParser.java    From flink with Apache License 2.0 6 votes vote down vote up
private List<RowType.RowField> parseRowFields(TokenType endToken) {
	List<RowType.RowField> fields = new ArrayList<>();
	boolean isFirst = true;
	while (!hasNextToken(endToken)) {
		if (isFirst) {
			isFirst = false;
		} else {
			nextToken(TokenType.LIST_SEPARATOR);
		}
		nextToken(TokenType.IDENTIFIER);
		final String name = tokenAsString();
		final LogicalType type = parseTypeWithNullability();
		if (hasNextToken(TokenType.LITERAL_STRING)) {
			nextToken(TokenType.LITERAL_STRING);
			final String description = tokenAsString();
			fields.add(new RowType.RowField(name, type, description));
		} else {
			fields.add(new RowType.RowField(name, type));
		}
	}
	return fields;
}
 
Example 2
Source File: LogicalTypeGeneralization.java    From flink with Apache License 2.0 6 votes vote down vote up
private static @Nullable LogicalType findCommonRowType(List<LogicalType> normalizedTypes) {
	final List<LogicalType> children = findCommonChildrenTypes(normalizedTypes);
	if (children == null) {
		return null;
	}
	final RowType firstType = (RowType) normalizedTypes.get(0);
	final List<RowType.RowField> newFields = IntStream.range(0, children.size())
		.mapToObj(pos -> {
			final LogicalType newType = children.get(pos);
			final RowType.RowField originalField = firstType.getFields().get(pos);
			if (originalField.getDescription().isPresent()) {
				return new RowType.RowField(
					originalField.getName(),
					newType,
					originalField.getDescription().get());
			} else {
				return new RowType.RowField(
					originalField.getName(),
					newType);
			}
		})
		.collect(Collectors.toList());
	return new RowType(newFields);
}
 
Example 3
Source File: HBaseTableSchema.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Construct a {@link HBaseTableSchema} from a {@link TableSchema}.
 */
public static HBaseTableSchema fromTableSchema(TableSchema schema) {
	HBaseTableSchema hbaseSchema = new HBaseTableSchema();
	RowType rowType = (RowType) schema.toPhysicalRowDataType().getLogicalType();
	for (RowType.RowField field : rowType.getFields()) {
		LogicalType fieldType = field.getType();
		if (fieldType.getTypeRoot() == LogicalTypeRoot.ROW) {
			RowType familyType = (RowType) fieldType;
			String familyName = field.getName();
			for (RowType.RowField qualifier : familyType.getFields()) {
				hbaseSchema.addColumn(
					familyName,
					qualifier.getName(),
					fromLogicalToDataType(qualifier.getType()));
			}
		} else if (fieldType.getChildren().size() == 0) {
			hbaseSchema.setRowKey(field.getName(), fromLogicalToDataType(fieldType));
		} else {
			throw new IllegalArgumentException(
				"Unsupported field type '" + fieldType + "' for HBase.");
		}
	}
	return hbaseSchema;
}
 
Example 4
Source File: LogicalTypeParser.java    From flink with Apache License 2.0 6 votes vote down vote up
private List<RowType.RowField> parseRowFields(TokenType endToken) {
	List<RowType.RowField> fields = new ArrayList<>();
	boolean isFirst = true;
	while (!hasNextToken(endToken)) {
		if (isFirst) {
			isFirst = false;
		} else {
			nextToken(TokenType.LIST_SEPARATOR);
		}
		nextToken(TokenType.IDENTIFIER);
		final String name = tokenAsString();
		final LogicalType type = parseTypeWithNullability();
		if (hasNextToken(TokenType.LITERAL_STRING)) {
			nextToken(TokenType.LITERAL_STRING);
			final String description = tokenAsString();
			fields.add(new RowType.RowField(name, type, description));
		} else {
			fields.add(new RowType.RowField(name, type));
		}
	}
	return fields;
}
 
Example 5
Source File: LogicalTypeMerging.java    From flink with Apache License 2.0 6 votes vote down vote up
private static @Nullable LogicalType findCommonRowType(List<LogicalType> normalizedTypes) {
	final List<LogicalType> children = findCommonChildrenTypes(normalizedTypes);
	if (children == null) {
		return null;
	}
	final RowType firstType = (RowType) normalizedTypes.get(0);
	final List<RowType.RowField> newFields = IntStream.range(0, children.size())
		.mapToObj(pos -> {
			final LogicalType newType = children.get(pos);
			final RowType.RowField originalField = firstType.getFields().get(pos);
			if (originalField.getDescription().isPresent()) {
				return new RowType.RowField(
					originalField.getName(),
					newType,
					originalField.getDescription().get());
			} else {
				return new RowType.RowField(
					originalField.getName(),
					newType);
			}
		})
		.collect(Collectors.toList());
	return new RowType(newFields);
}
 
Example 6
Source File: ArrowUtils.java    From flink with Apache License 2.0 6 votes vote down vote up
private static Field toArrowField(String fieldName, LogicalType logicalType) {
	FieldType fieldType = new FieldType(
		logicalType.isNullable(),
		logicalType.accept(LogicalTypeToArrowTypeConverter.INSTANCE),
		null);
	List<Field> children = null;
	if (logicalType instanceof ArrayType) {
		children = Collections.singletonList(toArrowField(
			"element", ((ArrayType) logicalType).getElementType()));
	} else if (logicalType instanceof RowType) {
		RowType rowType = (RowType) logicalType;
		children = new ArrayList<>(rowType.getFieldCount());
		for (RowType.RowField field : rowType.getFields()) {
			children.add(toArrowField(field.getName(), field.getType()));
		}
	}
	return new Field(fieldName, fieldType, children);
}
 
Example 7
Source File: PythonTypeUtils.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public FlinkFnApi.Schema.FieldType visit(RowType rowType) {
	FlinkFnApi.Schema.FieldType.Builder builder =
		FlinkFnApi.Schema.FieldType.newBuilder()
			.setTypeName(FlinkFnApi.Schema.TypeName.ROW)
			.setNullable(rowType.isNullable());

	FlinkFnApi.Schema.Builder schemaBuilder = FlinkFnApi.Schema.newBuilder();
	for (RowType.RowField field : rowType.getFields()) {
		schemaBuilder.addFields(
			FlinkFnApi.Schema.Field.newBuilder()
				.setName(field.getName())
				.setDescription(field.getDescription().orElse(EMPTY_STRING))
				.setType(field.getType().accept(this))
				.build());
	}
	builder.setRowSchema(schemaBuilder.build());
	return builder.build();
}
 
Example 8
Source File: ArrowSourceFunctionTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@BeforeClass
public static void init() {
	fieldTypes.add(new VarCharType());
	List<RowType.RowField> rowFields = new ArrayList<>();
	for (int i = 0; i < fieldTypes.size(); i++) {
		rowFields.add(new RowType.RowField("f" + i, fieldTypes.get(i)));
	}
	rowType = new RowType(rowFields);
	dataType = TypeConversions.fromLogicalToDataType(rowType);
	serializer = new RowDataSerializer(
		new ExecutionConfig(), fieldTypes.toArray(new LogicalType[0]));
	allocator = ArrowUtils.getRootAllocator().newChildAllocator("stdout", 0, Long.MAX_VALUE);
}
 
Example 9
Source File: PulsarSerializer.java    From pulsar-flink with Apache License 2.0 5 votes vote down vote up
private Function<Object, Object> newStructConverter(FieldsDataType dataType, Schema avroStruct) throws SchemaUtils.IncompatibleSchemaException {
    if (avroStruct.getType() != RECORD ||
            avroStruct.getFields().size() != dataType.getFieldDataTypes().size()) {
        throw new SchemaUtils.IncompatibleSchemaException(
                String.format("Cannot convert Flink type %s to Avro type %s.", dataType.toString(), avroStruct.toString(true)));
    }

    Map<String, DataType> fieldsType = dataType.getFieldDataTypes();
    List<RowType.RowField> fields = ((RowType) dataType.getLogicalType()).getFields();

    List<BiFunction<PositionedGetter, Integer, Object>> fieldConverters = new ArrayList<>();

    for (int i = 0; i < fields.size(); i++) {
        RowType.RowField rf = fields.get(i);
        DataType dt = fieldsType.get(rf.getName());
        Schema.Field at = avroStruct.getFields().get(i);
        fieldConverters.add(newConverter(dt, resolveNullableType(at.schema(), dt.getLogicalType().isNullable())));
    }
    int numFields = fieldsType.size();

    return row -> {
        GenericSchema<GenericRecord> pSchema = SchemaUtils.avroSchema2PulsarSchema(avroStruct);
        GenericRecordBuilder builder = pSchema.newRecordBuilder();
        Row rowX = (Row) row;

        for (int i = 0; i < numFields; i++) {
            if (rowX.getField(i) == null) {
                builder.set(pSchema.getFields().get(i), null);
            } else {
                builder.set(pSchema.getFields().get(i), fieldConverters.get(i).apply(new PositionedGetter(rowX), i));
            }
        }
        return (GenericAvroRecord) builder.build();
    };

}
 
Example 10
Source File: PythonTypeUtilsTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testLogicalTypeToProto() {
	List<RowType.RowField> rowFields = new ArrayList<>();
	rowFields.add(new RowType.RowField("f1", new BigIntType()));
	RowType rowType = new RowType(rowFields);
	FlinkFnApi.Schema.FieldType protoType =
		rowType.accept(new PythonTypeUtils.LogicalTypeToProtoTypeConverter());
	FlinkFnApi.Schema schema = protoType.getRowSchema();
	assertEquals(1, schema.getFieldsCount());
	assertEquals("f1", schema.getFields(0).getName());
	assertEquals(FlinkFnApi.Schema.TypeName.BIGINT, schema.getFields(0).getType().getTypeName());
}
 
Example 11
Source File: PythonTypeUtilsTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testLogicalTypeToBlinkTypeSerializer() {
	List<RowType.RowField> rowFields = new ArrayList<>();
	rowFields.add(new RowType.RowField("f1", new BigIntType()));
	RowType rowType = new RowType(rowFields);
	TypeSerializer baseSerializer = PythonTypeUtils.toBlinkTypeSerializer(rowType);
	assertTrue(baseSerializer instanceof RowDataSerializer);

	assertEquals(1, ((RowDataSerializer) baseSerializer).getArity());
}
 
Example 12
Source File: RowArrowSourceFunctionTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@BeforeClass
public static void init() {
	fieldTypes.add(new VarCharType());
	List<RowType.RowField> rowFields = new ArrayList<>();
	for (int i = 0; i < fieldTypes.size(); i++) {
		rowFields.add(new RowType.RowField("f" + i, fieldTypes.get(i)));
	}
	rowType = new RowType(rowFields);
	dataType = TypeConversions.fromLogicalToDataType(rowType);
	allocator = ArrowUtils.getRootAllocator().newChildAllocator("stdout", 0, Long.MAX_VALUE);
}
 
Example 13
Source File: AbstractPythonTableFunctionOperator.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public void open() throws Exception {
	List<RowType.RowField> udtfOutputDataFields = new ArrayList<>(
		outputType.getFields().subList(inputType.getFieldCount(), outputType.getFieldCount()));
	userDefinedFunctionOutputType = new RowType(udtfOutputDataFields);
	super.open();
}
 
Example 14
Source File: CsvRowSchemaConverter.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Convert {@link RowType} to {@link CsvSchema}.
 */
public static CsvSchema convert(RowType rowType) {
	Builder builder = new CsvSchema.Builder();
	List<RowType.RowField> fields = rowType.getFields();
	for (int i = 0; i < rowType.getFieldCount(); i++) {
		String fieldName = fields.get(i).getName();
		LogicalType fieldType = fields.get(i).getType();
		builder.addColumn(new Column(i, fieldName, convertType(fieldName, fieldType)));
	}
	return builder.build();
}
 
Example 15
Source File: LogicalTypeParser.java    From flink with Apache License 2.0 5 votes vote down vote up
private LogicalType parseRowType() {
	List<RowType.RowField> fields;
	// SQL standard notation
	if (hasNextToken(TokenType.BEGIN_PARAMETER)) {
		nextToken(TokenType.BEGIN_PARAMETER);
		fields = parseRowFields(TokenType.END_PARAMETER);
		nextToken(TokenType.END_PARAMETER);
	} else {
		nextToken(TokenType.BEGIN_SUBTYPE);
		fields = parseRowFields(TokenType.END_SUBTYPE);
		nextToken(TokenType.END_SUBTYPE);
	}
	return new RowType(fields);
}
 
Example 16
Source File: FlinkTypeVisitor.java    From iceberg with Apache License 2.0 5 votes vote down vote up
static <T> T visit(DataType dataType, FlinkTypeVisitor<T> visitor) {
  if (dataType instanceof FieldsDataType) {
    FieldsDataType fieldsType = (FieldsDataType) dataType;
    Map<String, DataType> fields = fieldsType.getFieldDataTypes();
    List<T> fieldResults = Lists.newArrayList();

    Preconditions.checkArgument(dataType.getLogicalType() instanceof RowType, "The logical type must be RowType");
    List<RowType.RowField> rowFields = ((RowType) dataType.getLogicalType()).getFields();
    // Make sure that we're traveling in the same order as the RowFields because the implementation of
    // FlinkTypeVisitor#fields may depends on the visit order, please see FlinkTypeToType#fields.
    for (RowType.RowField rowField : rowFields) {
      String name = rowField.getName();
      fieldResults.add(visit(fields.get(name), visitor));
    }

    return visitor.fields(fieldsType, fieldResults);
  } else if (dataType instanceof CollectionDataType) {
    CollectionDataType collectionType = (CollectionDataType) dataType;
    return visitor.collection(collectionType,
        visit(collectionType.getElementDataType(), visitor));
  } else if (dataType instanceof KeyValueDataType) {
    KeyValueDataType mapType = (KeyValueDataType) dataType;
    return visitor.map(mapType,
        visit(mapType.getKeyDataType(), visitor),
        visit(mapType.getValueDataType(), visitor));
  } else if (dataType instanceof AtomicDataType) {
    AtomicDataType atomic = (AtomicDataType) dataType;
    return visitor.atomic(atomic);
  } else {
    throw new UnsupportedOperationException("Unsupported data type: " + dataType);
  }
}
 
Example 17
Source File: RowDataArrowReaderWriterTest.java    From flink with Apache License 2.0 4 votes vote down vote up
@BeforeClass
public static void init() {
	fieldTypes.add(new TinyIntType());
	fieldTypes.add(new SmallIntType());
	fieldTypes.add(new IntType());
	fieldTypes.add(new BigIntType());
	fieldTypes.add(new BooleanType());
	fieldTypes.add(new FloatType());
	fieldTypes.add(new DoubleType());
	fieldTypes.add(new VarCharType());
	fieldTypes.add(new VarBinaryType());
	fieldTypes.add(new DecimalType(10, 3));
	fieldTypes.add(new DateType());
	fieldTypes.add(new TimeType(0));
	fieldTypes.add(new TimeType(2));
	fieldTypes.add(new TimeType(4));
	fieldTypes.add(new TimeType(8));
	fieldTypes.add(new LocalZonedTimestampType(0));
	fieldTypes.add(new LocalZonedTimestampType(2));
	fieldTypes.add(new LocalZonedTimestampType(4));
	fieldTypes.add(new LocalZonedTimestampType(8));
	fieldTypes.add(new TimestampType(0));
	fieldTypes.add(new TimestampType(2));
	fieldTypes.add(new TimestampType(4));
	fieldTypes.add(new TimestampType(8));
	fieldTypes.add(new ArrayType(new VarCharType()));
	rowFieldType = new RowType(Arrays.asList(
		new RowType.RowField("a", new IntType()),
		new RowType.RowField("b", new VarCharType()),
		new RowType.RowField("c", new ArrayType(new VarCharType())),
		new RowType.RowField("d", new TimestampType(2)),
		new RowType.RowField("e", new RowType(Arrays.asList(
			new RowType.RowField("e1", new IntType()),
			new RowType.RowField("e2", new VarCharType()))))));
	fieldTypes.add(rowFieldType);

	List<RowType.RowField> rowFields = new ArrayList<>();
	for (int i = 0; i < fieldTypes.size(); i++) {
		rowFields.add(new RowType.RowField("f" + i, fieldTypes.get(i)));
	}
	rowType = new RowType(rowFields);
	allocator = ArrowUtils.getRootAllocator().newChildAllocator("stdout", 0, Long.MAX_VALUE);
}
 
Example 18
Source File: ArrowUtilsTest.java    From flink with Apache License 2.0 4 votes vote down vote up
@BeforeClass
public static void init() {
	testFields = new ArrayList<>();
	testFields.add(Tuple7.of(
		"f1", new TinyIntType(), new ArrowType.Int(8, true), RowTinyIntWriter.class,
		TinyIntWriter.TinyIntWriterForRow.class, TinyIntFieldReader.class, ArrowTinyIntColumnVector.class));

	testFields.add(Tuple7.of("f2", new SmallIntType(), new ArrowType.Int(8 * 2, true),
		RowSmallIntWriter.class, SmallIntWriter.SmallIntWriterForRow.class, SmallIntFieldReader.class, ArrowSmallIntColumnVector.class));

	testFields.add(Tuple7.of("f3", new IntType(), new ArrowType.Int(8 * 4, true),
		RowIntWriter.class, IntWriter.IntWriterForRow.class, IntFieldReader.class, ArrowIntColumnVector.class));

	testFields.add(Tuple7.of("f4", new BigIntType(), new ArrowType.Int(8 * 8, true),
		RowBigIntWriter.class, BigIntWriter.BigIntWriterForRow.class, BigIntFieldReader.class, ArrowBigIntColumnVector.class));

	testFields.add(Tuple7.of("f5", new BooleanType(), new ArrowType.Bool(),
		RowBooleanWriter.class, BooleanWriter.BooleanWriterForRow.class, BooleanFieldReader.class, ArrowBooleanColumnVector.class));

	testFields.add(Tuple7.of("f6", new FloatType(), new ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE),
		RowFloatWriter.class, FloatWriter.FloatWriterForRow.class, FloatFieldReader.class, ArrowFloatColumnVector.class));

	testFields.add(Tuple7.of("f7", new DoubleType(), new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE),
		RowDoubleWriter.class, DoubleWriter.DoubleWriterForRow.class, DoubleFieldReader.class, ArrowDoubleColumnVector.class));

	testFields.add(Tuple7.of("f8", new VarCharType(), ArrowType.Utf8.INSTANCE,
		RowVarCharWriter.class, VarCharWriter.VarCharWriterForRow.class, VarCharFieldReader.class, ArrowVarCharColumnVector.class));

	testFields.add(Tuple7.of("f9", new VarBinaryType(), ArrowType.Binary.INSTANCE,
		RowVarBinaryWriter.class, VarBinaryWriter.VarBinaryWriterForRow.class, VarBinaryFieldReader.class, ArrowVarBinaryColumnVector.class));

	testFields.add(Tuple7.of("f10", new DecimalType(10, 3), new ArrowType.Decimal(10, 3),
		RowDecimalWriter.class, DecimalWriter.DecimalWriterForRow.class, DecimalFieldReader.class, ArrowDecimalColumnVector.class));

	testFields.add(Tuple7.of("f11", new DateType(), new ArrowType.Date(DateUnit.DAY),
		RowDateWriter.class, DateWriter.DateWriterForRow.class, DateFieldReader.class, ArrowDateColumnVector.class));

	testFields.add(Tuple7.of("f13", new TimeType(0), new ArrowType.Time(TimeUnit.SECOND, 32),
		RowTimeWriter.class, TimeWriter.TimeWriterForRow.class, TimeFieldReader.class, ArrowTimeColumnVector.class));

	testFields.add(Tuple7.of("f14", new TimeType(2), new ArrowType.Time(TimeUnit.MILLISECOND, 32),
		RowTimeWriter.class, TimeWriter.TimeWriterForRow.class, TimeFieldReader.class, ArrowTimeColumnVector.class));

	testFields.add(Tuple7.of("f15", new TimeType(4), new ArrowType.Time(TimeUnit.MICROSECOND, 64),
		RowTimeWriter.class, TimeWriter.TimeWriterForRow.class, TimeFieldReader.class, ArrowTimeColumnVector.class));

	testFields.add(Tuple7.of("f16", new TimeType(8), new ArrowType.Time(TimeUnit.NANOSECOND, 64),
		RowTimeWriter.class, TimeWriter.TimeWriterForRow.class, TimeFieldReader.class, ArrowTimeColumnVector.class));

	testFields.add(Tuple7.of("f17", new LocalZonedTimestampType(0), new ArrowType.Timestamp(TimeUnit.SECOND, null),
		RowTimestampWriter.class, TimestampWriter.TimestampWriterForRow.class, TimestampFieldReader.class, ArrowTimestampColumnVector.class));

	testFields.add(Tuple7.of("f18", new LocalZonedTimestampType(2), new ArrowType.Timestamp(TimeUnit.MILLISECOND, null),
		RowTimestampWriter.class, TimestampWriter.TimestampWriterForRow.class, TimestampFieldReader.class, ArrowTimestampColumnVector.class));

	testFields.add(Tuple7.of("f19", new LocalZonedTimestampType(4), new ArrowType.Timestamp(TimeUnit.MICROSECOND, null),
		RowTimestampWriter.class, TimestampWriter.TimestampWriterForRow.class, TimestampFieldReader.class, ArrowTimestampColumnVector.class));

	testFields.add(Tuple7.of("f20", new LocalZonedTimestampType(8), new ArrowType.Timestamp(TimeUnit.NANOSECOND, null),
		RowTimestampWriter.class, TimestampWriter.TimestampWriterForRow.class, TimestampFieldReader.class, ArrowTimestampColumnVector.class));

	testFields.add(Tuple7.of("f21", new TimestampType(0), new ArrowType.Timestamp(TimeUnit.SECOND, null),
		RowTimestampWriter.class, TimestampWriter.TimestampWriterForRow.class, TimestampFieldReader.class, ArrowTimestampColumnVector.class));

	testFields.add(Tuple7.of("f22", new TimestampType(2), new ArrowType.Timestamp(TimeUnit.MILLISECOND, null),
		RowTimestampWriter.class, TimestampWriter.TimestampWriterForRow.class, TimestampFieldReader.class, ArrowTimestampColumnVector.class));

	testFields.add(Tuple7.of("f23", new TimestampType(4), new ArrowType.Timestamp(TimeUnit.MICROSECOND, null),
		RowTimestampWriter.class, TimestampWriter.TimestampWriterForRow.class, TimestampFieldReader.class, ArrowTimestampColumnVector.class));

	testFields.add(Tuple7.of("f24", new TimestampType(8), new ArrowType.Timestamp(TimeUnit.NANOSECOND, null),
		RowTimestampWriter.class, TimestampWriter.TimestampWriterForRow.class, TimestampFieldReader.class, ArrowTimestampColumnVector.class));

	testFields.add(Tuple7.of("f25", new ArrayType(new VarCharType()), ArrowType.List.INSTANCE,
		RowArrayWriter.class, ArrayWriter.ArrayWriterForRow.class, ArrayFieldReader.class, ArrowArrayColumnVector.class));

	RowType rowFieldType = new RowType(Arrays.asList(
		new RowType.RowField("a", new IntType()),
		new RowType.RowField("b", new VarCharType()),
		new RowType.RowField("c", new ArrayType(new VarCharType())),
		new RowType.RowField("d", new TimestampType(2)),
		new RowType.RowField("e", new RowType((Arrays.asList(
			new RowType.RowField("e1", new IntType()),
			new RowType.RowField("e2", new VarCharType())))))));
	testFields.add(Tuple7.of("f26", rowFieldType, ArrowType.Struct.INSTANCE,
		RowRowWriter.class, RowWriter.RowWriterForRow.class, RowFieldReader.class, ArrowRowColumnVector.class));

	List<RowType.RowField> rowFields = new ArrayList<>();
	for (Tuple7<String, LogicalType, ArrowType, Class<?>, Class<?>, Class<?>, Class<?>> field : testFields) {
		rowFields.add(new RowType.RowField(field.f0, field.f1));
	}
	rowType = new RowType(rowFields);

	allocator = ArrowUtils.getRootAllocator().newChildAllocator("stdout", 0, Long.MAX_VALUE);
}
 
Example 19
Source File: PulsarDeserializer.java    From pulsar-flink with Apache License 2.0 4 votes vote down vote up
private BinFunction<RowUpdater, GenericRecord> getRecordWriter(Schema avroType, FieldsDataType sqlType, List<String> path) throws SchemaUtils.IncompatibleSchemaException {
    List<Integer> validFieldIndexes = new ArrayList<>();
    List<BinFunction<RowUpdater, Object>> fieldWriters = new ArrayList<>();

    int length = sqlType.getFieldDataTypes().size();
    List<RowType.RowField> fields = ((RowType) sqlType.getLogicalType()).getFields();
    Map<String, DataType> fieldsType = sqlType.getFieldDataTypes();

    for (int i = 0; i < length; i++) {
        RowType.RowField sqlField = fields.get(i);
        Schema.Field avroField = avroType.getField(sqlField.getName());
        if (avroField != null) {
            validFieldIndexes.add(avroField.pos());

            TriFunction<FlinkDataUpdater, Integer, Object> baseWriter = newWriter(
                    avroField.schema(), fieldsType.get(sqlField.getName()),
                    Stream.concat(path.stream(), Stream.of(sqlField.getName())).collect(Collectors.toList()));
            int ordinal = i;

            BinFunction<RowUpdater, Object> fieldWriter = (updater, value) -> {
                if (value == null) {
                    updater.setNullAt(ordinal);
                } else {
                    baseWriter.apply(updater, ordinal, value);
                }
            };

            fieldWriters.add(fieldWriter);

        } else if (!sqlField.getType().isNullable()) {
            throw new SchemaUtils.IncompatibleSchemaException(String.format(
                    "Cannot find non-nullable field in avro schema %s", avroType));
        }
    }

    return (rowUpdater, record) -> {
        for (int i = 0; i < validFieldIndexes.size(); i++) {
            fieldWriters.get(i).apply(rowUpdater, record.get(validFieldIndexes.get(i)));
        }
    };
}
 
Example 20
Source File: DataTypes.java    From flink with Apache License 2.0 3 votes vote down vote up
/**
 * Data type of a sequence of fields. A field consists of a field name, field type, and an optional
 * description. The most specific type of a row of a table is a row type. In this case, each column
 * of the row corresponds to the field of the row type that has the same ordinal position as the
 * column.
 *
 * <p>Compared to the SQL standard, an optional field description simplifies the handling with
 * complex structures.
 *
 * @see RowType
 */
public static DataType ROW(Field... fields) {
	final List<RowType.RowField> logicalFields = Stream.of(fields)
		.map(f -> Preconditions.checkNotNull(f, "Field definition must not be null."))
		.map(f -> new RowType.RowField(f.name, f.dataType.getLogicalType(), f.description))
		.collect(Collectors.toList());
	final Map<String, DataType> fieldDataTypes = Stream.of(fields)
		.collect(Collectors.toMap(f -> f.name, f -> f.dataType));
	return new FieldsDataType(new RowType(logicalFields), fieldDataTypes);
}