Java Code Examples for org.apache.flink.api.common.typeinfo.Types#ROW_NAMED

private static Map<String, String> hbaseTableProperties() {
	Map<String, String> properties = new HashMap<>();
	properties.put(CONNECTOR_TYPE, CONNECTOR_TYPE_VALUE_HBASE);
	properties.put(CONNECTOR_VERSION, CONNECTOR_VERSION_VALUE_143);
	properties.put(CONNECTOR_PROPERTY_VERSION, "1");
	properties.put(CONNECTOR_TABLE_NAME, TEST_TABLE_1);
	// get zk quorum from "hbase-site.xml" in classpath
	String hbaseZk = HBaseConfiguration.create().get(HConstants.ZOOKEEPER_QUORUM);
	properties.put(CONNECTOR_ZK_QUORUM, hbaseZk);
	// schema
	String[] columnNames = {FAMILY1, ROWKEY, FAMILY2, FAMILY3};
	TypeInformation<Row> f1 = Types.ROW_NAMED(new String[]{F1COL1}, Types.INT);
	TypeInformation<Row> f2 = Types.ROW_NAMED(new String[]{F2COL1, F2COL2}, Types.STRING, Types.LONG);
	TypeInformation<Row> f3 = Types.ROW_NAMED(new String[]{F3COL1, F3COL2, F3COL3}, Types.DOUBLE, Types.BOOLEAN, Types.STRING);
	TypeInformation[] columnTypes = new TypeInformation[]{f1, Types.INT, f2, f3};

	DescriptorProperties descriptorProperties = new DescriptorProperties(true);
	TableSchema tableSchema = new TableSchema(columnNames, columnTypes);
	descriptorProperties.putTableSchema(SCHEMA, tableSchema);
	descriptorProperties.putProperties(properties);
	return descriptorProperties.asMap();
}
 

@Test
public void testNestedSchema() throws IOException {
	final TypeInformation<Row> rowSchema = Types.ROW_NAMED(
		new String[] {"f1", "f2", "f3"},
		Types.INT, Types.BOOLEAN, Types.ROW(Types.INT, Types.DOUBLE));

	final Row row = new Row(3);
	row.setField(0, 42);
	row.setField(1, false);
	final Row nested = new Row(2);
	nested.setField(0, 22);
	nested.setField(1, 2.3);
	row.setField(2, nested);

	final Row resultRow = serializeAndDeserialize(rowSchema, row);
	assertEquals(row, resultRow);
}
 

@Test
public void testSerializationOfTwoRows() throws IOException {
	final TypeInformation<Row> rowSchema = Types.ROW_NAMED(
		new String[] {"f1", "f2", "f3"},
		Types.INT, Types.BOOLEAN, Types.STRING);

	final Row row1 = new Row(3);
	row1.setField(0, 1);
	row1.setField(1, true);
	row1.setField(2, "str");

	final JsonRowSerializationSchema serializationSchema = new JsonRowSerializationSchema(rowSchema);
	final JsonRowDeserializationSchema deserializationSchema = new JsonRowDeserializationSchema(rowSchema);

	byte[] bytes = serializationSchema.serialize(row1);
	assertEquals(row1, deserializationSchema.deserialize(bytes));

	final Row row2 = new Row(3);
	row2.setField(0, 10);
	row2.setField(1, false);
	row2.setField(2, "newStr");

	bytes = serializationSchema.serialize(row2);
	assertEquals(row2, deserializationSchema.deserialize(bytes));
}
 

@Test
public void testComplexSchema() throws Exception {
	final URL url = getClass().getClassLoader().getResource("complex-schema.json");
	Objects.requireNonNull(url);
	final String schema = FileUtils.readFileUtf8(new File(url.getFile()));
	final TypeInformation<?> result = JsonRowSchemaConverter.convert(schema);

	final TypeInformation<?> expected = Types.ROW_NAMED(
		new String[] {"fn", "familyName", "additionalName", "tuples", "honorificPrefix", "url",
			"email", "tel", "sound", "org"},
		Types.STRING, Types.STRING, Types.BOOLEAN, Types.ROW(Types.BIG_DEC, Types.STRING, Types.STRING, Types.STRING),
		Types.OBJECT_ARRAY(Types.STRING), Types.STRING, Types.ROW_NAMED(new String[] {"type", "value"}, Types.STRING, Types.STRING),
		Types.ROW_NAMED(new String[] {"type", "value"}, Types.BIG_DEC, Types.STRING), Types.VOID,
		Types.ROW_NAMED(new String[] {"organizationUnit"}, Types.ROW()));

	assertEquals(expected, result);
}
 

@Test
public void testGetReturnType() throws Exception {

	OrcTableSource orc = OrcTableSource.builder()
		.path(getPath(TEST_FILE_NESTED))
		.forOrcSchema(TEST_SCHEMA_NESTED)
		.build();

	TypeInformation<Row> returnType = orc.getReturnType();
	assertNotNull(returnType);
	assertTrue(returnType instanceof RowTypeInfo);
	RowTypeInfo rowType = (RowTypeInfo) returnType;

	TypeInformation<Row> expected = Types.ROW_NAMED(getNestedFieldNames(), getNestedFieldTypes());
	assertEquals(expected, rowType);
}
 

@Test
public void testGetReturnType() throws Exception {

	OrcTableSource orc = OrcTableSource.builder()
		.path(getPath(TEST_FILE_NESTED))
		.forOrcSchema(TEST_SCHEMA_NESTED)
		.build();

	TypeInformation<Row> returnType = orc.getReturnType();
	assertNotNull(returnType);
	assertTrue(returnType instanceof RowTypeInfo);
	RowTypeInfo rowType = (RowTypeInfo) returnType;

	TypeInformation<Row> expected = Types.ROW_NAMED(getNestedFieldNames(), getNestedFieldTypes());
	assertEquals(expected, rowType);
}
 

@Test
public void testNestedSchema() {
	final TypeInformation<Row> rowSchema = Types.ROW_NAMED(
		new String[] {"f1", "f2", "f3"},
		Types.INT, Types.BOOLEAN, Types.ROW(Types.INT, Types.DOUBLE));

	final Row row = new Row(3);
	row.setField(0, 42);
	row.setField(1, false);
	final Row nested = new Row(2);
	nested.setField(0, 22);
	nested.setField(1, 2.3);
	row.setField(2, nested);

	final JsonRowSerializationSchema serializationSchema = new JsonRowSerializationSchema.Builder(rowSchema)
		.build();
	final JsonRowDeserializationSchema deserializationSchema = new JsonRowDeserializationSchema.Builder(rowSchema)
		.build();

	assertThat(row, whenSerializedWith(serializationSchema)
		.andDeserializedWith(deserializationSchema)
		.equalsTo(row));
}
 

@Test
public void testComplexSchema() throws Exception {
	final URL url = getClass().getClassLoader().getResource("complex-schema.json");
	Objects.requireNonNull(url);
	final String schema = FileUtils.readFileUtf8(new File(url.getFile()));
	final TypeInformation<?> result = JsonRowSchemaConverter.convert(schema);

	final TypeInformation<?> expected = Types.ROW_NAMED(
		new String[] {"fn", "familyName", "additionalName", "tuples", "honorificPrefix", "url",
			"email", "tel", "sound", "org"},
		Types.STRING, Types.STRING, Types.BOOLEAN, Types.ROW(Types.BIG_DEC, Types.STRING, Types.STRING, Types.STRING),
		Types.OBJECT_ARRAY(Types.STRING), Types.STRING, Types.ROW_NAMED(new String[] {"type", "value"}, Types.STRING, Types.STRING),
		Types.ROW_NAMED(new String[] {"type", "value"}, Types.BIG_DEC, Types.STRING), Types.VOID,
		Types.ROW_NAMED(new String[] {"organizationUnit"}, Types.ROW()));

	assertEquals(expected, result);
}
 

@Test
public void testReferenceSchema() throws Exception {
	final URL url = getClass().getClassLoader().getResource("reference-schema.json");
	Objects.requireNonNull(url);
	final String schema = FileUtils.readFileUtf8(new File(url.getFile()));
	final TypeInformation<?> result = JsonRowSchemaConverter.convert(schema);

	final TypeInformation<?> expected = Types.ROW_NAMED(
		new String[] {"billing_address", "shipping_address", "optional_address"},
		Types.ROW_NAMED(new String[] {"street_address", "city", "state"}, Types.STRING, Types.STRING, Types.STRING),
		Types.ROW_NAMED(new String[] {"street_address", "city", "state"}, Types.STRING, Types.STRING, Types.STRING),
		Types.ROW_NAMED(new String[] {"street_address", "city", "state"}, Types.STRING, Types.STRING, Types.STRING));

	assertEquals(expected, result);
}
 

@Test
public void testSerializationOfTwoRows() throws IOException {
	final TypeInformation<Row> rowSchema = Types.ROW_NAMED(
		new String[] {"f1", "f2", "f3"},
		Types.INT, Types.BOOLEAN, Types.STRING);

	final Row row1 = new Row(3);
	row1.setField(0, 1);
	row1.setField(1, true);
	row1.setField(2, "str");

	final JsonRowSerializationSchema serializationSchema = new JsonRowSerializationSchema.Builder(rowSchema)
		.build();
	final JsonRowDeserializationSchema deserializationSchema = new JsonRowDeserializationSchema.Builder(rowSchema)
		.build();

	byte[] bytes = serializationSchema.serialize(row1);
	assertEquals(row1, deserializationSchema.deserialize(bytes));

	final Row row2 = new Row(3);
	row2.setField(0, 10);
	row2.setField(1, false);
	row2.setField(2, "newStr");

	bytes = serializationSchema.serialize(row2);
	assertEquals(row2, deserializationSchema.deserialize(bytes));
}
 

@Test(expected = RuntimeException.class)
public void testSerializeRowWithInvalidNumberOfFields() {
	final TypeInformation<Row> rowSchema = Types.ROW_NAMED(
		new String[] {"f1", "f2", "f3"},
		Types.INT, Types.BOOLEAN, Types.STRING);

	final Row row = new Row(1);
	row.setField(0, 1);

	final JsonRowSerializationSchema serializationSchema = new JsonRowSerializationSchema(rowSchema);
	serializationSchema.serialize(row);
}
 

private static TypeInformation<?> convertToRowTypeInfo(FieldsDataType fieldsDataType) {
	final RowType rowType = (RowType) fieldsDataType.getLogicalType();

	final String[] fieldNames = rowType.getFields()
		.stream()
		.map(RowType.RowField::getName)
		.toArray(String[]::new);

	final TypeInformation<?>[] fieldTypes = fieldsDataType.getChildren()
		.stream()
		.map(LegacyTypeInfoDataTypeConverter::toLegacyTypeInfo)
		.toArray(TypeInformation[]::new);

	return Types.ROW_NAMED(fieldNames, fieldTypes);
}
 

private static TypeInformation<Row> convertObject(String location, JsonNode node, JsonNode root) {
	// validate properties
	if (!node.has(PROPERTIES)) {
		return Types.ROW();
	}
	if (!node.isObject()) {
		throw new IllegalArgumentException(
			"Invalid '" + PROPERTIES + "' property for object type in node: " + location);
	}
	final JsonNode props = node.get(PROPERTIES);
	final String[] names = new String[props.size()];
	final TypeInformation<?>[] types = new TypeInformation[props.size()];

	final Iterator<Map.Entry<String, JsonNode>> fieldIter = props.fields();
	int i = 0;
	while (fieldIter.hasNext()) {
		final Map.Entry<String, JsonNode> subNode = fieldIter.next();

		// set field name
		names[i] = subNode.getKey();

		// set type
		types[i] = convertType(location + '/' + subNode.getKey(), subNode.getValue(), root);

		i++;
	}

	// validate that object does not contain additional properties
	if (node.has(ADDITIONAL_PROPERTIES) && node.get(ADDITIONAL_PROPERTIES).isBoolean() &&
			node.get(ADDITIONAL_PROPERTIES).asBoolean()) {
		throw new IllegalArgumentException(
			"An object must not allow additional properties in node: " + location);
	}

	return Types.ROW_NAMED(names, types);
}
 

/**
 * Creates a result. Might start threads or opens sockets so every created result must be closed.
 */
public <T> DynamicResult<T> createResult(Environment env, TableSchema schema, ExecutionConfig config) {

	final TypeInformation<Row> outputType = Types.ROW_NAMED(schema.getFieldNames(), schema.getFieldTypes());

	if (env.getExecution().isStreamingExecution()) {
		// determine gateway address (and port if possible)
		final InetAddress gatewayAddress = getGatewayAddress(env.getDeployment());
		final int gatewayPort = getGatewayPort(env.getDeployment());

		if (env.getExecution().isChangelogMode()) {
			return new ChangelogCollectStreamResult<>(outputType, config, gatewayAddress, gatewayPort);
		} else {
			return new MaterializedCollectStreamResult<>(
				outputType,
				config,
				gatewayAddress,
				gatewayPort,
				env.getExecution().getMaxTableResultRows());
		}

	} else {
		// Batch Execution
		if (!env.getExecution().isTableMode()) {
			throw new SqlExecutionException("Results of batch queries can only be served in table mode.");
		}
		return new MaterializedCollectBatchResult<>(outputType, config);
	}
}
 

private TypeInformation[] getNestedFieldTypes() {
	return new TypeInformation[]{
		Types.BOOLEAN, Types.BYTE, Types.SHORT, Types.INT, Types.LONG, Types.FLOAT, Types.DOUBLE,
		PrimitiveArrayTypeInfo.BYTE_PRIMITIVE_ARRAY_TYPE_INFO, Types.STRING,
		Types.ROW_NAMED(
			new String[]{"list"},
			ObjectArrayTypeInfo.getInfoFor(
				Types.ROW_NAMED(
					new String[]{"int1", "string1"},
					Types.INT, Types.STRING
				)
			)
		),
		ObjectArrayTypeInfo.getInfoFor(
			Types.ROW_NAMED(
				new String[]{"int1", "string1"},
				Types.INT, Types.STRING
			)
		),
		new MapTypeInfo<>(
			Types.STRING,
			Types.ROW_NAMED(
				new String[]{"int1", "string1"},
				Types.INT, Types.STRING
			)
		)
	};
}
 

private static TypeInformation<?> convertToTypeInfo(Schema schema) {
	switch (schema.getType()) {
		case RECORD:
			final List<Schema.Field> fields = schema.getFields();

			final TypeInformation<?>[] types = new TypeInformation<?>[fields.size()];
			final String[] names = new String[fields.size()];
			for (int i = 0; i < fields.size(); i++) {
				final Schema.Field field = fields.get(i);
				types[i] = convertToTypeInfo(field.schema());
				names[i] = field.name();
			}
			return Types.ROW_NAMED(names, types);
		case ENUM:
			return Types.STRING;
		case ARRAY:
			// result type might either be ObjectArrayTypeInfo or BasicArrayTypeInfo for Strings
			return Types.OBJECT_ARRAY(convertToTypeInfo(schema.getElementType()));
		case MAP:
			return Types.MAP(Types.STRING, convertToTypeInfo(schema.getValueType()));
		case UNION:
			final Schema actualSchema;
			if (schema.getTypes().size() == 2 && schema.getTypes().get(0).getType() == Schema.Type.NULL) {
				actualSchema = schema.getTypes().get(1);
			} else if (schema.getTypes().size() == 2 && schema.getTypes().get(1).getType() == Schema.Type.NULL) {
				actualSchema = schema.getTypes().get(0);
			} else if (schema.getTypes().size() == 1) {
				actualSchema = schema.getTypes().get(0);
			} else {
				// use Kryo for serialization
				return Types.GENERIC(Object.class);
			}
			return convertToTypeInfo(actualSchema);
		case FIXED:
			// logical decimal type
			if (schema.getLogicalType() instanceof LogicalTypes.Decimal) {
				return Types.BIG_DEC;
			}
			// convert fixed size binary data to primitive byte arrays
			return Types.PRIMITIVE_ARRAY(Types.BYTE);
		case STRING:
			// convert Avro's Utf8/CharSequence to String
			return Types.STRING;
		case BYTES:
			// logical decimal type
			if (schema.getLogicalType() instanceof LogicalTypes.Decimal) {
				return Types.BIG_DEC;
			}
			return Types.PRIMITIVE_ARRAY(Types.BYTE);
		case INT:
			// logical date and time type
			final org.apache.avro.LogicalType logicalType = schema.getLogicalType();
			if (logicalType == LogicalTypes.date()) {
				return Types.SQL_DATE;
			} else if (logicalType == LogicalTypes.timeMillis()) {
				return Types.SQL_TIME;
			}
			return Types.INT;
		case LONG:
			// logical timestamp type
			if (schema.getLogicalType() == LogicalTypes.timestampMillis()) {
				return Types.SQL_TIMESTAMP;
			}
			return Types.LONG;
		case FLOAT:
			return Types.FLOAT;
		case DOUBLE:
			return Types.DOUBLE;
		case BOOLEAN:
			return Types.BOOLEAN;
		case NULL:
			return Types.VOID;
	}
	throw new IllegalArgumentException("Unsupported Avro type '" + schema.getType() + "'.");
}
 

private void validateUserSchema(TypeInformation<?> actual) {
	final TypeInformation<Row> address = Types.ROW_NAMED(
		new String[]{
			"num",
			"street",
			"city",
			"state",
			"zip"},
		Types.INT,
		Types.STRING,
		Types.STRING,
		Types.STRING,
		Types.STRING);

	final TypeInformation<Row> user = Types.ROW_NAMED(
		new String[] {
			"name",
			"favorite_number",
			"favorite_color",
			"type_long_test",
			"type_double_test",
			"type_null_test",
			"type_bool_test",
			"type_array_string",
			"type_array_boolean",
			"type_nullable_array",
			"type_enum",
			"type_map",
			"type_fixed",
			"type_union",
			"type_nested",
			"type_bytes",
			"type_date",
			"type_time_millis",
			"type_time_micros",
			"type_timestamp_millis",
			"type_timestamp_micros",
			"type_decimal_bytes",
			"type_decimal_fixed"},
		Types.STRING,
		Types.INT,
		Types.STRING,
		Types.LONG,
		Types.DOUBLE,
		Types.VOID,
		Types.BOOLEAN,
		Types.OBJECT_ARRAY(Types.STRING),
		Types.OBJECT_ARRAY(Types.BOOLEAN),
		Types.OBJECT_ARRAY(Types.STRING),
		Types.STRING,
		Types.MAP(Types.STRING, Types.LONG),
		Types.PRIMITIVE_ARRAY(Types.BYTE),
		Types.GENERIC(Object.class),
		address,
		Types.PRIMITIVE_ARRAY(Types.BYTE),
		Types.SQL_DATE,
		Types.SQL_TIME,
		Types.INT,
		Types.SQL_TIMESTAMP,
		Types.LONG,
		Types.BIG_DEC,
		Types.BIG_DEC);

	assertEquals(user, actual);

	final RowTypeInfo userRowInfo = (RowTypeInfo) user;
	assertTrue(userRowInfo.schemaEquals(actual));
}
 

@Override
public TypeInformation<Row> getRecordType() {
  return Types.ROW_NAMED(fieldNames, fieldTypes);
}
 

@Override
public TypeInformation<Row> getOutputType() {
	return Types.ROW_NAMED(fieldNames, fieldTypes);
}
 

/**
 * Converts a table schema into a (nested) type information describing a {@link Row}.
 */
public TypeInformation<Row> toRowType() {
	return Types.ROW_NAMED(fieldNames, fieldTypes);
}