org.apache.flink.table.types.FieldsDataType Java Examples

@Override
public DataType visit(FieldsDataType fieldsDataType) {
	final List<DataType> fieldDataTypes = fieldsDataType.getChildren();
	if (logicalType.getTypeRoot() == LogicalTypeRoot.ROW) {
		final List<String> fieldNames = getFieldNames(logicalType);
		DataTypes.Field[] fields = IntStream.range(0, fieldDataTypes.size())
			.mapToObj(i -> {
				final DataType oldFieldType = fieldDataTypes.get(i);
				final DataType newFieldType = oldFieldType.accept(
					new DataTypePrecisionFixer(logicalType.getChildren().get(i)));
				return DataTypes.FIELD(fieldNames.get(i), newFieldType);
			})
			.toArray(DataTypes.Field[]::new);
		return DataTypes.ROW(fields).bridgedTo(fieldsDataType.getConversionClass());
	}
	throw new UnsupportedOperationException("Unsupported logical type : " + logicalType);
}
 

private static TableSchema expandCompositeType(FieldsDataType dataType) {
	DataType[] fieldDataTypes = dataType.getChildren().toArray(new DataType[0]);
	return dataType.getLogicalType().accept(new LogicalTypeDefaultVisitor<TableSchema>() {
		@Override
		public TableSchema visit(RowType rowType) {
			return expandCompositeType(rowType, fieldDataTypes);
		}

		@Override
		public TableSchema visit(StructuredType structuredType) {
			return expandCompositeType(structuredType, fieldDataTypes);
		}

		@Override
		public TableSchema visit(DistinctType distinctType) {
			return distinctType.getSourceType().accept(this);
		}

		@Override
		protected TableSchema defaultMethod(LogicalType logicalType) {
			throw new IllegalArgumentException("Expected a composite type");
		}
	});
}
 

private static DataType getPojoWithRawSelfReferenceDataType() {
	final StructuredType.Builder builder = StructuredType.newBuilder(PojoWithRawSelfReference.class);
	builder.attributes(
		Arrays.asList(
			new StructuredAttribute(
				"integer",
				new IntType()),
			new StructuredAttribute(
				"reference",
				new TypeInformationRawType<>(new GenericTypeInfo<>(PojoWithRawSelfReference.class)))));
	builder.setFinal(true);
	builder.setInstantiable(true);
	final StructuredType structuredType = builder.build();

	final List<DataType> fieldDataTypes = Arrays.asList(
		DataTypes.INT(),
		DataTypes.RAW(new GenericTypeInfo<>(PojoWithRawSelfReference.class))
	);

	return new FieldsDataType(structuredType, PojoWithRawSelfReference.class, fieldDataTypes);
}
 

private static DataType getInnerTupleDataType() {
	final StructuredType.Builder builder = StructuredType.newBuilder(Tuple2.class);
	builder.attributes(
		Arrays.asList(
			new StructuredAttribute(
				"f0",
				new VarCharType(VarCharType.MAX_LENGTH)),
			new StructuredAttribute(
				"f1",
				new BooleanType())));
	builder.setFinal(true);
	builder.setInstantiable(true);
	final StructuredType structuredType = builder.build();

	final List<DataType> fieldDataTypes = Arrays.asList(
		DataTypes.STRING(),
		DataTypes.BOOLEAN()
	);

	return new FieldsDataType(structuredType, Tuple2.class, fieldDataTypes);
}
 

public static FieldsDataType pulsarSourceSchema(SchemaInfo si) throws IncompatibleSchemaException {
    List<DataTypes.Field> mainSchema = new ArrayList<>();
    DataType dataType = si2SqlType(si);
    if (dataType instanceof FieldsDataType) {
        FieldsDataType fieldsDataType = (FieldsDataType) dataType;
        RowType rowType = (RowType) fieldsDataType.getLogicalType();
        rowType.getFieldNames().stream()
                .map(fieldName -> DataTypes.FIELD(fieldName, fieldsDataType.getFieldDataTypes().get(fieldName)))
                .forEach(mainSchema::add);
    } else {
        mainSchema.add(DataTypes.FIELD("value", dataType));
    }

    mainSchema.addAll(METADATA_FIELDS);
    return (FieldsDataType) DataTypes.ROW(mainSchema.toArray(new DataTypes.Field[0]));
}
 

private static DataType getOuterTupleDataType() {
	final StructuredType.Builder builder = StructuredType.newBuilder(Tuple2.class);
	builder.attributes(
		Arrays.asList(
			new StructuredAttribute(
				"f0",
				new IntType()),
			new StructuredAttribute(
				"f1",
				getInnerTupleDataType().getLogicalType())));
	builder.setFinal(true);
	builder.setInstantiable(true);
	final StructuredType structuredType = builder.build();

	final List<DataType> fieldDataTypes = Arrays.asList(
		DataTypes.INT(),
		getInnerTupleDataType()
	);

	return new FieldsDataType(structuredType, Tuple2.class, fieldDataTypes);
}
 

/**
 * Testing data type shared with the Scala tests.
 */
public static DataType getPojoWithCustomOrderDataType(Class<?> pojoClass) {
	final StructuredType.Builder builder = StructuredType.newBuilder(pojoClass);
	builder.attributes(
		Arrays.asList(
			new StructuredAttribute(
				"z",
				new BigIntType()),
			new StructuredAttribute(
				"y",
				new BooleanType()),
			new StructuredAttribute(
				"x",
				new IntType())));
	builder.setFinal(true);
	builder.setInstantiable(true);
	final StructuredType structuredType = builder.build();

	final List<DataType> fieldDataTypes = Arrays.asList(
		DataTypes.BIGINT(),
		DataTypes.BOOLEAN(),
		DataTypes.INT()
	);

	return new FieldsDataType(structuredType, pojoClass, fieldDataTypes);
}
 

/**
 * Testing data type shared with the Scala tests.
 */
static DataType getComplexPojoDataType(Class<?> complexPojoClass, Class<?> simplePojoClass) {
	final StructuredType.Builder builder = StructuredType.newBuilder(complexPojoClass);
	builder.attributes(
		Arrays.asList(
			new StructuredAttribute(
				"mapField",
				new MapType(new VarCharType(VarCharType.MAX_LENGTH), new IntType())),
			new StructuredAttribute(
				"simplePojoField",
				getSimplePojoDataType(simplePojoClass).getLogicalType()),
			new StructuredAttribute(
				"someObject",
				new TypeInformationRawType<>(new GenericTypeInfo<>(Object.class)))));
	builder.setFinal(true);
	builder.setInstantiable(true);
	final StructuredType structuredType = builder.build();

	final List<DataType> fieldDataTypes = Arrays.asList(
		DataTypes.MAP(DataTypes.STRING(), DataTypes.INT()),
		getSimplePojoDataType(simplePojoClass),
		DataTypes.RAW(new GenericTypeInfo<>(Object.class))
	);

	return new FieldsDataType(structuredType, complexPojoClass, fieldDataTypes);
}
 

/**
 * Testing data type shared with the Scala tests.
 */
static DataType getSimplePojoDataType(Class<?> simplePojoClass) {
	final StructuredType.Builder builder = StructuredType.newBuilder(simplePojoClass);
	builder.attributes(
		Arrays.asList(
			new StructuredAttribute("intField", new IntType(true)),
			new StructuredAttribute("primitiveBooleanField", new BooleanType(false)),
			new StructuredAttribute("primitiveIntField", new IntType(false)),
			new StructuredAttribute("stringField", new VarCharType(VarCharType.MAX_LENGTH))));
	builder.setFinal(true);
	builder.setInstantiable(true);
	final StructuredType structuredType = builder.build();

	final List<DataType> fieldDataTypes = Arrays.asList(
		DataTypes.INT(),
		DataTypes.BOOLEAN().notNull().bridgedTo(boolean.class),
		DataTypes.INT().notNull().bridgedTo(int.class),
		DataTypes.STRING()
	);

	return new FieldsDataType(structuredType, simplePojoClass, fieldDataTypes);
}
 

private static DataType structuredType(
		String typeName,
		List<DataType> fieldDataTypes,
		StructuredComparision comparision) {
	return new FieldsDataType(
		StructuredType.newBuilder(ObjectIdentifier.of("cat", "db", typeName))
			.attributes(
				IntStream.range(0, fieldDataTypes.size())
					.mapToObj(idx -> new StructuredType.StructuredAttribute(
						"f" + idx,
						fieldDataTypes.get(idx).getLogicalType()))
					.collect(Collectors.toList())
			)
			.comparision(comparision)
			.build(),
		fieldDataTypes
	);
}
 

@Test
public void testJDBCWithFilter() {
	Map<String, String> properties = getBasicProperties();
	properties.put("connector.driver", "org.apache.derby.jdbc.EmbeddedDriver");
	properties.put("connector.username", "user");
	properties.put("connector.password", "pass");

	final TableSource<?> actual = ((JDBCTableSource) TableFactoryService
		.find(StreamTableSourceFactory.class, properties)
		.createStreamTableSource(properties))
		.projectFields(new int[] {0, 2});

	Map<String, DataType> projectedFields = ((FieldsDataType) actual.getProducedDataType()).getFieldDataTypes();
	assertEquals(projectedFields.get("aaa"), DataTypes.INT());
	assertNull(projectedFields.get("bbb"));
	assertEquals(projectedFields.get("ccc"), DataTypes.DOUBLE());
}
 

static <T> T visit(DataType dataType, FlinkTypeVisitor<T> visitor) {
  if (dataType instanceof FieldsDataType) {
    FieldsDataType fieldsType = (FieldsDataType) dataType;
    Map<String, DataType> fields = fieldsType.getFieldDataTypes();
    List<T> fieldResults = Lists.newArrayList();

    Preconditions.checkArgument(dataType.getLogicalType() instanceof RowType, "The logical type must be RowType");
    List<RowType.RowField> rowFields = ((RowType) dataType.getLogicalType()).getFields();
    // Make sure that we're traveling in the same order as the RowFields because the implementation of
    // FlinkTypeVisitor#fields may depends on the visit order, please see FlinkTypeToType#fields.
    for (RowType.RowField rowField : rowFields) {
      String name = rowField.getName();
      fieldResults.add(visit(fields.get(name), visitor));
    }

    return visitor.fields(fieldsType, fieldResults);
  } else if (dataType instanceof CollectionDataType) {
    CollectionDataType collectionType = (CollectionDataType) dataType;
    return visitor.collection(collectionType,
        visit(collectionType.getElementDataType(), visitor));
  } else if (dataType instanceof KeyValueDataType) {
    KeyValueDataType mapType = (KeyValueDataType) dataType;
    return visitor.map(mapType,
        visit(mapType.getKeyDataType(), visitor),
        visit(mapType.getValueDataType(), visitor));
  } else if (dataType instanceof AtomicDataType) {
    AtomicDataType atomic = (AtomicDataType) dataType;
    return visitor.atomic(atomic);
  } else {
    throw new UnsupportedOperationException("Unsupported data type: " + dataType);
  }
}
 

@Override
public TypeInformation<Row> getProducedType() {
    if (typeInformation == null) {
        try (PulsarMetadataReader reader = new PulsarMetadataReader(adminUrl, clientConfigurationData, "", caseInsensitiveParams, -1, -1)) {
            List<String> topics = reader.getTopics();
            FieldsDataType schema = reader.getSchema(topics);
            typeInformation = (TypeInformation<Row>) LegacyTypeInfoDataTypeConverter.toLegacyTypeInfo(schema);
        } catch (Exception e) {
            log.error("Failed to get schema for source with exception {}", ExceptionUtils.stringifyException(e));
            typeInformation = null;
        }
    }

    return typeInformation;
}
 

/**
 * Convert the flink table schema to apache iceberg schema.
 */
public static Schema convert(TableSchema schema) {
  Preconditions.checkArgument(schema.toRowDataType() instanceof FieldsDataType, "Should be FieldsDataType");

  FieldsDataType root = (FieldsDataType) schema.toRowDataType();
  Type converted = FlinkTypeVisitor.visit(root, new FlinkTypeToType(root));

  return new Schema(converted.asStructType().fields());
}
 

private static TypeInformation<?> convertToRowTypeInfo(FieldsDataType fieldsDataType) {
	final RowType rowType = (RowType) fieldsDataType.getLogicalType();

	final String[] fieldNames = rowType.getFields()
		.stream()
		.map(RowType.RowField::getName)
		.toArray(String[]::new);

	final TypeInformation<?>[] fieldTypes = fieldsDataType.getChildren()
		.stream()
		.map(LegacyTypeInfoDataTypeConverter::toLegacyTypeInfo)
		.toArray(TypeInformation[]::new);

	return Types.ROW_NAMED(fieldNames, fieldTypes);
}
 

@Override
public DataType visit(RowType rowType) {
	final List<DataType> fieldDataTypes = rowType.getFields()
		.stream()
		.map(f -> f.getType().accept(this))
		.collect(Collectors.toList());
	return new FieldsDataType(
		rowType,
		fieldDataTypes);
}
 

@Override
public DataType visit(StructuredType structuredType) {
	final List<DataType> attributeDataTypes = structuredType.getAttributes()
		.stream()
		.map(a -> a.getType().accept(this))
		.collect(Collectors.toList());
	return new FieldsDataType(
		structuredType,
		attributeDataTypes);
}
 

@Test
public void testExpandStructuredType() {
	StructuredType logicalType = StructuredType.newBuilder(ObjectIdentifier.of("catalog", "database", "type"))
		.attributes(Arrays.asList(
			new StructuredType.StructuredAttribute("f0", DataTypes.INT().getLogicalType()),
			new StructuredType.StructuredAttribute("f1", DataTypes.STRING().getLogicalType()),
			new StructuredType.StructuredAttribute("f2", DataTypes.TIMESTAMP(5).getLogicalType()),
			new StructuredType.StructuredAttribute("f3", DataTypes.TIMESTAMP(3).getLogicalType())
		))
		.build();

	List<DataType> dataTypes = Arrays.asList(
		DataTypes.INT(),
		DataTypes.STRING(),
		DataTypes.TIMESTAMP(5).bridgedTo(Timestamp.class),
		DataTypes.TIMESTAMP(3));
	FieldsDataType dataType = new FieldsDataType(logicalType, dataTypes);

	TableSchema schema = DataTypeUtils.expandCompositeTypeToSchema(dataType);

	assertThat(
		schema,
		equalTo(
			TableSchema.builder()
				.field("f0", INT())
				.field("f1", STRING())
				.field("f2", TIMESTAMP(5).bridgedTo(Timestamp.class))
				.field("f3", TIMESTAMP(3).bridgedTo(LocalDateTime.class))
				.build()));
}
 

@Test
public void testExpandDistinctType() {
	FieldsDataType dataType = (FieldsDataType) ROW(
		FIELD("f0", INT()),
		FIELD("f1", STRING()),
		FIELD("f2", TIMESTAMP(5).bridgedTo(Timestamp.class)),
		FIELD("f3", TIMESTAMP(3)));

	LogicalType originalLogicalType = dataType.getLogicalType();
	DistinctType distinctLogicalType = DistinctType.newBuilder(
		ObjectIdentifier.of("catalog", "database", "type"),
		originalLogicalType)
		.build();
	DataType distinctDataType = new FieldsDataType(distinctLogicalType, dataType.getChildren());

	TableSchema schema = DataTypeUtils.expandCompositeTypeToSchema(distinctDataType);

	assertThat(
		schema,
		equalTo(
			TableSchema.builder()
				.field("f0", INT())
				.field("f1", STRING())
				.field("f2", TIMESTAMP(5).bridgedTo(Timestamp.class))
				.field("f3", TIMESTAMP(3).bridgedTo(LocalDateTime.class))
				.build()));
}
 

@Test
public void testIsCompositeTypeStructuredType() {
	StructuredType logicalType = StructuredType.newBuilder(ObjectIdentifier.of("catalog", "database", "type"))
		.attributes(Arrays.asList(
			new StructuredType.StructuredAttribute("f0", DataTypes.INT().getLogicalType()),
			new StructuredType.StructuredAttribute("f1", DataTypes.STRING().getLogicalType())
		))
		.build();

	List<DataType> fieldDataTypes = Arrays.asList(DataTypes.INT(), DataTypes.STRING());
	FieldsDataType dataType = new FieldsDataType(logicalType, fieldDataTypes);
	boolean isCompositeType = LogicalTypeChecks.isCompositeType(dataType.getLogicalType());

	assertThat(isCompositeType, is(true));
}
 

private Optional<ResolvedExpression> convertRowToExpectedType(
		ResolvedExpression sourceExpression,
		FieldsDataType targetDataType,
		ExpressionResolver.PostResolverFactory postResolverFactory) {
	List<DataType> targetDataTypes = targetDataType.getChildren();
	List<ResolvedExpression> resolvedChildren = sourceExpression.getResolvedChildren();

	if (resolvedChildren.size() != targetDataTypes.size()) {
		return Optional.empty();
	}

	ResolvedExpression[] castedChildren = new ResolvedExpression[resolvedChildren.size()];
	for (int i = 0; i < resolvedChildren.size(); i++) {
		boolean typesMatch = resolvedChildren.get(i)
			.getOutputDataType()
			.getLogicalType()
			.equals(targetDataTypes.get(i).getLogicalType());
		if (typesMatch) {
			castedChildren[i] = resolvedChildren.get(i);
		}

		ResolvedExpression child = resolvedChildren.get(i);
		DataType targetChildDataType = targetDataTypes.get(i);

		Optional<ResolvedExpression> castedChild = convertToExpectedType(
			child,
			targetChildDataType,
			postResolverFactory);

		if (!castedChild.isPresent()) {
			return Optional.empty();
		} else {
			castedChildren[i] = castedChild.get();
		}
	}

	return Optional.of(postResolverFactory.row(targetDataType, castedChildren));
}
 

private TableSchema inferTableSchema() {
    if (providedSchema.isPresent()) {
        return providedSchema.get();
    } else {
        try {
            PulsarMetadataReader reader = new PulsarMetadataReader(adminUrl, new ClientConfigurationData(), "", caseInsensitiveParams, -1, -1);
            List<String> topics = reader.getTopics();
            FieldsDataType schema = reader.getSchema(topics);
            return SchemaUtils.toTableSchema(schema);
        } catch (PulsarClientException | PulsarAdminException | SchemaUtils.IncompatibleSchemaException e) {
            log.error("Failed to fetch table schema", adminUrl);
            throw new RuntimeException(e);
        }
    }
}
 

@Override
public DataType visit(StructuredType structuredType) {
	final Map<String, DataType> attributeDataTypes = structuredType.getAttributes()
		.stream()
		.collect(Collectors.toMap(StructuredAttribute::getName, a -> a.getType().accept(this)));
	return new FieldsDataType(
		structuredType,
		attributeDataTypes);
}
 

public TableSchema getTableSchema(ObjectPath objectPath) throws PulsarAdminException {
    String topicName = objectPath2TopicName(objectPath);
    FieldsDataType fieldsDataType = null;
    try {
        fieldsDataType = getSchema(Collections.singletonList(topicName));
    } catch (IncompatibleSchemaException e) {
        throw new PulsarAdminException(e);
    }
    return SchemaUtils.toTableSchema(fieldsDataType);
}
 

@Override
public DataType visit(RowType rowType) {
	final Map<String, DataType> fieldDataTypes = rowType.getFields()
		.stream()
		.collect(Collectors.toMap(RowType.RowField::getName, f -> f.getType().accept(this)));
	return new FieldsDataType(
		rowType,
		fieldDataTypes);
}
 

private static TypeInformation<?> convertToRowTypeInfo(FieldsDataType fieldsDataType) {
	final RowType rowType = (RowType) fieldsDataType.getLogicalType();

	final String[] fieldNames = rowType.getFields()
		.stream()
		.map(RowType.RowField::getName)
		.toArray(String[]::new);

	final TypeInformation<?>[] fieldTypes = Stream.of(fieldNames)
		.map(name -> fieldsDataType.getFieldDataTypes().get(name))
		.map(LegacyTypeInfoDataTypeConverter::toLegacyTypeInfo)
		.toArray(TypeInformation[]::new);

	return Types.ROW_NAMED(fieldNames, fieldTypes);
}
 

public PulsarSerializer(DataType flinkType, boolean nullable) {
    this.flinkType = flinkType;
    this.nullable = nullable;

    try {
        this.rootAvroType = SchemaUtils.sqlType2AvroSchema(flinkType);

        Schema actualAvroType = resolveNullableType(rootAvroType, nullable);
        Function<Object, Object> baseConverter;
        if (flinkType instanceof FieldsDataType) {
            FieldsDataType st = (FieldsDataType) flinkType;
            baseConverter = newStructConverter(st, actualAvroType);
        } else {
            BiFunction<PositionedGetter, Integer, Object> converter = singleValueConverter(flinkType, actualAvroType);
            baseConverter =
                    data -> converter.apply(new PositionedGetter((Row) data), 0);
        }

        if (nullable) {
            this.converter = data -> {
                if (data == null) {
                    return null;
                } else {
                    return baseConverter.apply(data);
                }
            };
        } else {
            this.converter = baseConverter;
        }

    } catch (SchemaUtils.IncompatibleSchemaException e) {
        log.error("Failed to create serializer while converting flink type to avro type");
        throw new RuntimeException(e);
    }
}
 

private Function<Object, Object> newStructConverter(FieldsDataType dataType, Schema avroStruct) throws SchemaUtils.IncompatibleSchemaException {
    if (avroStruct.getType() != RECORD ||
            avroStruct.getFields().size() != dataType.getFieldDataTypes().size()) {
        throw new SchemaUtils.IncompatibleSchemaException(
                String.format("Cannot convert Flink type %s to Avro type %s.", dataType.toString(), avroStruct.toString(true)));
    }

    Map<String, DataType> fieldsType = dataType.getFieldDataTypes();
    List<RowType.RowField> fields = ((RowType) dataType.getLogicalType()).getFields();

    List<BiFunction<PositionedGetter, Integer, Object>> fieldConverters = new ArrayList<>();

    for (int i = 0; i < fields.size(); i++) {
        RowType.RowField rf = fields.get(i);
        DataType dt = fieldsType.get(rf.getName());
        Schema.Field at = avroStruct.getFields().get(i);
        fieldConverters.add(newConverter(dt, resolveNullableType(at.schema(), dt.getLogicalType().isNullable())));
    }
    int numFields = fieldsType.size();

    return row -> {
        GenericSchema<GenericRecord> pSchema = SchemaUtils.avroSchema2PulsarSchema(avroStruct);
        GenericRecordBuilder builder = pSchema.newRecordBuilder();
        Row rowX = (Row) row;

        for (int i = 0; i < numFields; i++) {
            if (rowX.getField(i) == null) {
                builder.set(pSchema.getFields().get(i), null);
            } else {
                builder.set(pSchema.getFields().get(i), fieldConverters.get(i).apply(new PositionedGetter(rowX), i));
            }
        }
        return (GenericAvroRecord) builder.build();
    };

}
 

public static TableSchema toTableSchema(FieldsDataType schema) {
    RowType rt = (RowType) schema.getLogicalType();
    List<DataType> fieldTypes = rt.getFieldNames().stream().map(fn -> schema.getFieldDataTypes().get(fn)).collect(Collectors.toList());

    return TableSchema.builder().fields(
            rt.getFieldNames().toArray(new String[0]), fieldTypes.toArray(new DataType[0])).build();
}
 

public static org.apache.pulsar.client.api.Schema sqlType2PulsarSchema(DataType flinkType) throws IncompatibleSchemaException {
    if (flinkType instanceof AtomicDataType) {
        LogicalTypeRoot type = flinkType.getLogicalType().getTypeRoot();
        switch (type) {
            case BOOLEAN:
                return BooleanSchema.of();
            case VARBINARY:
                return BytesSchema.of();
            case DATE:
                return DateSchema.of();
            case VARCHAR:
                return org.apache.pulsar.client.api.Schema.STRING;
            case TIMESTAMP_WITHOUT_TIME_ZONE:
                return TimestampSchema.of();
            case TINYINT:
                return ByteSchema.of();
            case DOUBLE:
                return DoubleSchema.of();
            case FLOAT:
                return FloatSchema.of();
            case INTEGER:
                return IntSchema.of();
            case BIGINT:
                return LongSchema.of();
            case SMALLINT:
                return ShortSchema.of();
            default:
                throw new IncompatibleSchemaException(String.format("%s is not supported by Pulsar yet", flinkType.toString()), null);
        }
    } else if (flinkType instanceof FieldsDataType) {
        return avroSchema2PulsarSchema(sqlType2AvroSchema(flinkType));
    }
    throw new IncompatibleSchemaException(String.format("%s is not supported by Pulsar yet", flinkType.toString()), null);
}