Java Code Examples for org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper#writeValueAsBytes()

The following examples show how to use org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper#writeValueAsBytes() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: JSONKeyValueDeserializationSchemaTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testDeserializeWithoutValue() throws Exception {
	ObjectMapper mapper = new ObjectMapper();
	ObjectNode initialKey = mapper.createObjectNode();
	initialKey.put("index", 4);
	byte[] serializedKey = mapper.writeValueAsBytes(initialKey);

	byte[] serializedValue = null;

	JSONKeyValueDeserializationSchema schema = new JSONKeyValueDeserializationSchema(false);
	ObjectNode deserializedValue = schema.deserialize(newConsumerRecord(serializedKey, serializedValue));

	Assert.assertTrue(deserializedValue.get("metadata") == null);
	Assert.assertEquals(4, deserializedValue.get("key").get("index").asInt());
	Assert.assertTrue(deserializedValue.get("value") == null);
}
 
Example 2
Source File: JSONKeyValueDeserializationSchemaTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testDeserializeWithMetadata() throws Exception {
	ObjectMapper mapper = new ObjectMapper();
	ObjectNode initialKey = mapper.createObjectNode();
	initialKey.put("index", 4);
	byte[] serializedKey = mapper.writeValueAsBytes(initialKey);

	ObjectNode initialValue = mapper.createObjectNode();
	initialValue.put("word", "world");
	byte[] serializedValue = mapper.writeValueAsBytes(initialValue);

	JSONKeyValueDeserializationSchema schema = new JSONKeyValueDeserializationSchema(true);
	final ConsumerRecord<byte[], byte[]> consumerRecord =
			newConsumerRecord("topic#1", 3, 4L, serializedKey, serializedValue);
	ObjectNode deserializedValue = schema.deserialize(consumerRecord);

	Assert.assertEquals(4, deserializedValue.get("key").get("index").asInt());
	Assert.assertEquals("world", deserializedValue.get("value").get("word").asText());
	Assert.assertEquals("topic#1", deserializedValue.get("metadata").get("topic").asText());
	Assert.assertEquals(4, deserializedValue.get("metadata").get("offset").asInt());
	Assert.assertEquals(3, deserializedValue.get("metadata").get("partition").asInt());
}
 
Example 3
Source File: JSONKeyValueDeserializationSchemaTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testDeserializeWithoutValue() throws Exception {
	ObjectMapper mapper = new ObjectMapper();
	ObjectNode initialKey = mapper.createObjectNode();
	initialKey.put("index", 4);
	byte[] serializedKey = mapper.writeValueAsBytes(initialKey);

	byte[] serializedValue = null;

	JSONKeyValueDeserializationSchema schema = new JSONKeyValueDeserializationSchema(false);
	ObjectNode deserializedValue = schema.deserialize(newConsumerRecord(serializedKey, serializedValue));

	Assert.assertTrue(deserializedValue.get("metadata") == null);
	Assert.assertEquals(4, deserializedValue.get("key").get("index").asInt());
	Assert.assertTrue(deserializedValue.get("value") == null);
}
 
Example 4
Source File: JSONKeyValueDeserializationSchemaTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testDeserializeWithoutKey() throws Exception {
	ObjectMapper mapper = new ObjectMapper();
	byte[] serializedKey = null;

	ObjectNode initialValue = mapper.createObjectNode();
	initialValue.put("word", "world");
	byte[] serializedValue = mapper.writeValueAsBytes(initialValue);

	JSONKeyValueDeserializationSchema schema = new JSONKeyValueDeserializationSchema(false);
	ObjectNode deserializedValue = schema.deserialize(newConsumerRecord(serializedKey, serializedValue));

	Assert.assertTrue(deserializedValue.get("metadata") == null);
	Assert.assertTrue(deserializedValue.get("key") == null);
	Assert.assertEquals("world", deserializedValue.get("value").get("word").asText());
}
 
Example 5
Source File: JsonRowDataSerDeSchemaTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testSerDeSQLTimestampFormat() throws Exception{
	RowType rowType = (RowType) ROW(
		FIELD("timestamp3", TIMESTAMP(3)),
		FIELD("timestamp9", TIMESTAMP(9))
	).getLogicalType();

	JsonRowDataDeserializationSchema deserializationSchema = new JsonRowDataDeserializationSchema(
		rowType, new RowDataTypeInfo(rowType), false, false, TimestampFormat.SQL);
	JsonRowDataSerializationSchema serializationSchema = new JsonRowDataSerializationSchema(rowType, TimestampFormat.SQL);

	ObjectMapper objectMapper = new ObjectMapper();

	ObjectNode root = objectMapper.createObjectNode();
	root.put("timestamp3", "1990-10-14 12:12:43.123");
	root.put("timestamp9", "1990-10-14 12:12:43.123456789");
	byte[] serializedJson = objectMapper.writeValueAsBytes(root);
	RowData rowData = deserializationSchema.deserialize(serializedJson);
	byte[] actual = serializationSchema.serialize(rowData);
	assertEquals(new String(serializedJson), new String(actual));
}
 
Example 6
Source File: JSONKeyValueDeserializationSchemaTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testDeserializeWithMetadata() throws Exception {
	ObjectMapper mapper = new ObjectMapper();
	ObjectNode initialKey = mapper.createObjectNode();
	initialKey.put("index", 4);
	byte[] serializedKey = mapper.writeValueAsBytes(initialKey);

	ObjectNode initialValue = mapper.createObjectNode();
	initialValue.put("word", "world");
	byte[] serializedValue = mapper.writeValueAsBytes(initialValue);

	JSONKeyValueDeserializationSchema schema = new JSONKeyValueDeserializationSchema(true);
	final ConsumerRecord<byte[], byte[]> consumerRecord =
			newConsumerRecord("topic#1", 3, 4L, serializedKey, serializedValue);
	ObjectNode deserializedValue = schema.deserialize(consumerRecord);

	Assert.assertEquals(4, deserializedValue.get("key").get("index").asInt());
	Assert.assertEquals("world", deserializedValue.get("value").get("word").asText());
	Assert.assertEquals("topic#1", deserializedValue.get("metadata").get("topic").asText());
	Assert.assertEquals(4, deserializedValue.get("metadata").get("offset").asInt());
	Assert.assertEquals(3, deserializedValue.get("metadata").get("partition").asInt());
}
 
Example 7
Source File: JSONKeyValueDeserializationSchemaTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Test
public void testDeserializeWithoutMetadata() throws Exception {
	ObjectMapper mapper = new ObjectMapper();
	ObjectNode initialKey = mapper.createObjectNode();
	initialKey.put("index", 4);
	byte[] serializedKey = mapper.writeValueAsBytes(initialKey);

	ObjectNode initialValue = mapper.createObjectNode();
	initialValue.put("word", "world");
	byte[] serializedValue = mapper.writeValueAsBytes(initialValue);

	JSONKeyValueDeserializationSchema schema = new JSONKeyValueDeserializationSchema(false);
	ObjectNode deserializedValue = schema.deserialize(newConsumerRecord(serializedKey, serializedValue));

	Assert.assertTrue(deserializedValue.get("metadata") == null);
	Assert.assertEquals(4, deserializedValue.get("key").get("index").asInt());
	Assert.assertEquals("world", deserializedValue.get("value").get("word").asText());
}
 
Example 8
Source File: JSONKeyValueDeserializationSchemaTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testDeserializeWithoutMetadata() throws Exception {
	ObjectMapper mapper = new ObjectMapper();
	ObjectNode initialKey = mapper.createObjectNode();
	initialKey.put("index", 4);
	byte[] serializedKey = mapper.writeValueAsBytes(initialKey);

	ObjectNode initialValue = mapper.createObjectNode();
	initialValue.put("word", "world");
	byte[] serializedValue = mapper.writeValueAsBytes(initialValue);

	JSONKeyValueDeserializationSchema schema = new JSONKeyValueDeserializationSchema(false);
	ObjectNode deserializedValue = schema.deserialize(newConsumerRecord(serializedKey, serializedValue));

	Assert.assertTrue(deserializedValue.get("metadata") == null);
	Assert.assertEquals(4, deserializedValue.get("key").get("index").asInt());
	Assert.assertEquals("world", deserializedValue.get("value").get("word").asText());
}
 
Example 9
Source File: JSONKeyValueDeserializationSchemaTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Test
public void testDeserializeWithMetadata() throws Exception {
	ObjectMapper mapper = new ObjectMapper();
	ObjectNode initialKey = mapper.createObjectNode();
	initialKey.put("index", 4);
	byte[] serializedKey = mapper.writeValueAsBytes(initialKey);

	ObjectNode initialValue = mapper.createObjectNode();
	initialValue.put("word", "world");
	byte[] serializedValue = mapper.writeValueAsBytes(initialValue);

	JSONKeyValueDeserializationSchema schema = new JSONKeyValueDeserializationSchema(true);
	final ConsumerRecord<byte[], byte[]> consumerRecord =
			newConsumerRecord("topic#1", 3, 4L, serializedKey, serializedValue);
	ObjectNode deserializedValue = schema.deserialize(consumerRecord);

	Assert.assertEquals(4, deserializedValue.get("key").get("index").asInt());
	Assert.assertEquals("world", deserializedValue.get("value").get("word").asText());
	Assert.assertEquals("topic#1", deserializedValue.get("metadata").get("topic").asText());
	Assert.assertEquals(4, deserializedValue.get("metadata").get("offset").asInt());
	Assert.assertEquals(3, deserializedValue.get("metadata").get("partition").asInt());
}
 
Example 10
Source File: JSONKeyValueDeserializationSchemaTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Test
public void testDeserializeWithoutValue() throws Exception {
	ObjectMapper mapper = new ObjectMapper();
	ObjectNode initialKey = mapper.createObjectNode();
	initialKey.put("index", 4);
	byte[] serializedKey = mapper.writeValueAsBytes(initialKey);

	byte[] serializedValue = null;

	JSONKeyValueDeserializationSchema schema = new JSONKeyValueDeserializationSchema(false);
	ObjectNode deserializedValue = schema.deserialize(newConsumerRecord(serializedKey, serializedValue));

	Assert.assertTrue(deserializedValue.get("metadata") == null);
	Assert.assertEquals(4, deserializedValue.get("key").get("index").asInt());
	Assert.assertTrue(deserializedValue.get("value") == null);
}
 
Example 11
Source File: JSONKeyValueDeserializationSchemaTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Test
public void testDeserializeWithoutKey() throws Exception {
	ObjectMapper mapper = new ObjectMapper();
	byte[] serializedKey = null;

	ObjectNode initialValue = mapper.createObjectNode();
	initialValue.put("word", "world");
	byte[] serializedValue = mapper.writeValueAsBytes(initialValue);

	JSONKeyValueDeserializationSchema schema = new JSONKeyValueDeserializationSchema(false);
	ObjectNode deserializedValue = schema.deserialize(newConsumerRecord(serializedKey, serializedValue));

	Assert.assertTrue(deserializedValue.get("metadata") == null);
	Assert.assertTrue(deserializedValue.get("key") == null);
	Assert.assertEquals("world", deserializedValue.get("value").get("word").asText());
}
 
Example 12
Source File: JsonNodeDeserializationSchemaTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Test
public void testDeserialize() throws IOException {
	ObjectMapper mapper = new ObjectMapper();
	ObjectNode initialValue = mapper.createObjectNode();
	initialValue.put("key", 4).put("value", "world");
	byte[] serializedValue = mapper.writeValueAsBytes(initialValue);

	JsonNodeDeserializationSchema schema = new JsonNodeDeserializationSchema();
	ObjectNode deserializedValue = schema.deserialize(serializedValue);

	assertEquals(4, deserializedValue.get("key").asInt());
	assertEquals("world", deserializedValue.get("value").asText());
}
 
Example 13
Source File: JsonRowDeserializationSchemaTest.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Tests deserialization with non-existing field name.
 */
@Test
public void testMissingNode() throws Exception {
	ObjectMapper objectMapper = new ObjectMapper();

	// Root
	ObjectNode root = objectMapper.createObjectNode();
	root.put("id", 123123123);
	byte[] serializedJson = objectMapper.writeValueAsBytes(root);

	TypeInformation<Row> rowTypeInformation = Types.ROW_NAMED(
		new String[]{"name"},
		Types.STRING);

	JsonRowDeserializationSchema deserializationSchema =
		new JsonRowDeserializationSchema.Builder(rowTypeInformation)
			.build();

	Row row = new Row(1);
	assertThat(serializedJson,
		whenDeserializedWith(deserializationSchema).equalsTo(row));

	deserializationSchema = new JsonRowDeserializationSchema.Builder(rowTypeInformation)
		.failOnMissingField()
		.build();

	assertThat(serializedJson,
		whenDeserializedWith(deserializationSchema)
			.failsWithException(hasCause(instanceOf(IllegalStateException.class))));
}
 
Example 14
Source File: JsonRowDeserializationSchemaTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
/**
 * Tests simple deserialization using type information.
 */
@Test
public void testTypeInfoDeserialization() throws Exception {
	long id = 1238123899121L;
	String name = "asdlkjasjkdla998y1122";
	byte[] bytes = new byte[1024];
	ThreadLocalRandom.current().nextBytes(bytes);

	ObjectMapper objectMapper = new ObjectMapper();

	// Root
	ObjectNode root = objectMapper.createObjectNode();
	root.put("id", id);
	root.put("name", name);
	root.put("bytes", bytes);

	byte[] serializedJson = objectMapper.writeValueAsBytes(root);

	JsonRowDeserializationSchema deserializationSchema = new JsonRowDeserializationSchema(
		Types.ROW_NAMED(
			new String[] { "id", "name", "bytes" },
			Types.LONG, Types.STRING, Types.PRIMITIVE_ARRAY(Types.BYTE))
	);

	Row deserialized = deserializationSchema.deserialize(serializedJson);

	assertEquals(3, deserialized.getArity());
	assertEquals(id, deserialized.getField(0));
	assertEquals(name, deserialized.getField(1));
	assertArrayEquals(bytes, (byte[]) deserialized.getField(2));
}
 
Example 15
Source File: JsonRowDeserializationSchemaTest.java    From flink with Apache License 2.0 4 votes vote down vote up
/**
 * Tests deserialization with non-existing field name.
 */
@Test
public void testMissingNode() throws Exception {
	ObjectMapper objectMapper = new ObjectMapper();

	// Root
	ObjectNode root = objectMapper.createObjectNode();
	root.put("id", 123123123);
	byte[] serializedJson = objectMapper.writeValueAsBytes(root);

	TypeInformation<Row> rowTypeInformation = Types.ROW_NAMED(
		new String[]{"name"},
		Types.STRING);

	JsonRowDeserializationSchema deserializationSchema =
		new JsonRowDeserializationSchema.Builder(rowTypeInformation)
			.build();

	Row row = new Row(1);
	assertThat(serializedJson,
		whenDeserializedWith(deserializationSchema).equalsTo(row));

	deserializationSchema = new JsonRowDeserializationSchema.Builder(rowTypeInformation)
		.failOnMissingField()
		.build();

	assertThat(serializedJson,
		whenDeserializedWith(deserializationSchema)
			.failsWithException(hasCause(instanceOf(IllegalStateException.class))));

	// ignore-parse-errors ignores missing field exception too
	deserializationSchema = new JsonRowDeserializationSchema.Builder(rowTypeInformation)
		.ignoreParseErrors()
		.build();
	assertThat(serializedJson,
		whenDeserializedWith(deserializationSchema).equalsTo(row));

	thrown.expect(IllegalArgumentException.class);
	thrown.expectMessage("JSON format doesn't support failOnMissingField and ignoreParseErrors are both true");
	new JsonRowDeserializationSchema.Builder(rowTypeInformation)
		.failOnMissingField()
		.ignoreParseErrors()
		.build();
}
 
Example 16
Source File: JsonRowDeserializationSchemaTest.java    From flink with Apache License 2.0 4 votes vote down vote up
@Test
public void testSchemaDeserialization() throws Exception {
	final BigDecimal id = BigDecimal.valueOf(1238123899121L);
	final String name = "asdlkjasjkdla998y1122";
	final byte[] bytes = new byte[1024];
	ThreadLocalRandom.current().nextBytes(bytes);
	final BigDecimal[] numbers = new BigDecimal[] {
		BigDecimal.valueOf(1), BigDecimal.valueOf(2), BigDecimal.valueOf(3)};
	final String[] strings = new String[] {"one", "two", "three"};

	final ObjectMapper objectMapper = new ObjectMapper();

	// Root
	ObjectNode root = objectMapper.createObjectNode();
	root.put("id", id.longValue());
	root.putNull("idOrNull");
	root.put("name", name);
	root.put("date", "1990-10-14");
	root.put("time", "12:12:43Z");
	root.put("timestamp", "1990-10-14T12:12:43Z");
	root.put("bytes", bytes);
	root.putArray("numbers").add(1).add(2).add(3);
	root.putArray("strings").add("one").add("two").add("three");
	root.putObject("nested").put("booleanField", true).put("decimalField", 12);

	final byte[] serializedJson = objectMapper.writeValueAsBytes(root);

	JsonRowDeserializationSchema deserializationSchema = new JsonRowDeserializationSchema.Builder(
		"{" +
		"    type: 'object'," +
		"    properties: {" +
		"         id: { type: 'integer' }," +
		"         idOrNull: { type: ['integer', 'null'] }," +
		"         name: { type: 'string' }," +
		"         date: { type: 'string', format: 'date' }," +
		"         time: { type: 'string', format: 'time' }," +
		"         timestamp: { type: 'string', format: 'date-time' }," +
		"         bytes: { type: 'string', contentEncoding: 'base64' }," +
		"         numbers: { type: 'array', items: { type: 'integer' } }," +
		"         strings: { type: 'array', items: { type: 'string' } }," +
		"         nested: { " +
		"             type: 'object'," +
		"             properties: { " +
		"                 booleanField: { type: 'boolean' }," +
		"                 decimalField: { type: 'number' }" +
		"             }" +
		"         }" +
		"    }" +
		"}").build();

	final Row expected = new Row(10);
	expected.setField(0, id);
	expected.setField(1, null);
	expected.setField(2, name);
	expected.setField(3, Date.valueOf("1990-10-14"));
	expected.setField(4, Time.valueOf("12:12:43"));
	expected.setField(5, Timestamp.valueOf("1990-10-14 12:12:43"));
	expected.setField(6, bytes);
	expected.setField(7, numbers);
	expected.setField(8, strings);
	final Row nestedRow = new Row(2);
	nestedRow.setField(0, true);
	nestedRow.setField(1, BigDecimal.valueOf(12));
	expected.setField(9, nestedRow);

	assertThat(serializedJson, whenDeserializedWith(deserializationSchema).equalsTo(expected));
}
 
Example 17
Source File: JsonRowDeserializationSchemaTest.java    From flink with Apache License 2.0 4 votes vote down vote up
/**
 * Tests simple deserialization using type information.
 */
@Test
public void testTypeInfoDeserialization() throws Exception {
	long id = 1238123899121L;
	String name = "asdlkjasjkdla998y1122";
	byte[] bytes = new byte[1024];
	ThreadLocalRandom.current().nextBytes(bytes);
	Timestamp timestamp = Timestamp.valueOf("1990-10-14 12:12:43");
	Date date = Date.valueOf("1990-10-14");
	Time time = Time.valueOf("12:12:43");

	Map<String, Long> map = new HashMap<>();
	map.put("flink", 123L);

	Map<String, Map<String, Integer>> nestedMap = new HashMap<>();
	Map<String, Integer> innerMap = new HashMap<>();
	innerMap.put("key", 234);
	nestedMap.put("inner_map", innerMap);

	ObjectMapper objectMapper = new ObjectMapper();

	// Root
	ObjectNode root = objectMapper.createObjectNode();
	root.put("id", id);
	root.put("name", name);
	root.put("bytes", bytes);
	root.put("date1", "1990-10-14");
	root.put("date2", "1990-10-14");
	root.put("time1", "12:12:43Z");
	root.put("time2", "12:12:43Z");
	root.put("timestamp1", "1990-10-14T12:12:43Z");
	root.put("timestamp2", "1990-10-14T12:12:43Z");
	root.putObject("map").put("flink", 123);
	root.putObject("map2map").putObject("inner_map").put("key", 234);

	byte[] serializedJson = objectMapper.writeValueAsBytes(root);

	JsonRowDeserializationSchema deserializationSchema = new JsonRowDeserializationSchema.Builder(
		Types.ROW_NAMED(
			new String[]{"id", "name", "bytes", "date1", "date2",
				"time1", "time2", "timestamp1", "timestamp2", "map", "map2map"},
			Types.LONG, Types.STRING, Types.PRIMITIVE_ARRAY(Types.BYTE),
			Types.SQL_DATE, Types.LOCAL_DATE, Types.SQL_TIME, Types.LOCAL_TIME,
			Types.SQL_TIMESTAMP, Types.LOCAL_DATE_TIME,
			Types.MAP(Types.STRING, Types.LONG),
			Types.MAP(Types.STRING, Types.MAP(Types.STRING, Types.INT)))
	).build();

	Row row = new Row(11);
	row.setField(0, id);
	row.setField(1, name);
	row.setField(2, bytes);
	row.setField(3, date);
	row.setField(4, date.toLocalDate());
	row.setField(5, time);
	row.setField(6, time.toLocalTime());
	row.setField(7, timestamp);
	row.setField(8, timestamp.toLocalDateTime());
	row.setField(9, map);
	row.setField(10, nestedMap);

	assertThat(serializedJson, whenDeserializedWith(deserializationSchema).equalsTo(row));
}
 
Example 18
Source File: JsonRowDataSerDeSchemaTest.java    From flink with Apache License 2.0 4 votes vote down vote up
/**
 * Tests the deserialization slow path,
 * e.g. convert into string and use {@link Double#parseDouble(String)}.
 */
@Test
public void testSlowDeserialization() throws Exception {
	Random random = new Random();
	boolean bool = random.nextBoolean();
	int integer = random.nextInt();
	long bigint = random.nextLong();
	double doubleValue = random.nextDouble();
	float floatValue = random.nextFloat();

	ObjectMapper objectMapper = new ObjectMapper();
	ObjectNode root = objectMapper.createObjectNode();
	root.put("bool", String.valueOf(bool));
	root.put("int", String.valueOf(integer));
	root.put("bigint", String.valueOf(bigint));
	root.put("double1", String.valueOf(doubleValue));
	root.put("double2", new BigDecimal(doubleValue));
	root.put("float1", String.valueOf(floatValue));
	root.put("float2", new BigDecimal(floatValue));

	byte[] serializedJson = objectMapper.writeValueAsBytes(root);

	DataType dataType = ROW(
		FIELD("bool", BOOLEAN()),
		FIELD("int", INT()),
		FIELD("bigint", BIGINT()),
		FIELD("double1", DOUBLE()),
		FIELD("double2", DOUBLE()),
		FIELD("float1", FLOAT()),
		FIELD("float2", FLOAT())
	);
	RowType rowType = (RowType) dataType.getLogicalType();

	JsonRowDataDeserializationSchema deserializationSchema = new JsonRowDataDeserializationSchema(
		rowType, new RowDataTypeInfo(rowType), false, false,  TimestampFormat.ISO_8601);

	Row expected = new Row(7);
	expected.setField(0, bool);
	expected.setField(1, integer);
	expected.setField(2, bigint);
	expected.setField(3, doubleValue);
	expected.setField(4, doubleValue);
	expected.setField(5, floatValue);
	expected.setField(6, floatValue);

	RowData rowData = deserializationSchema.deserialize(serializedJson);
	Row actual = convertToExternal(rowData, dataType);
	assertEquals(expected, actual);
}
 
Example 19
Source File: JsonRowDeserializationSchemaTest.java    From flink with Apache License 2.0 4 votes vote down vote up
@Test
public void testSchemaDeserialization() throws Exception {
	final BigDecimal id = BigDecimal.valueOf(1238123899121L);
	final String name = "asdlkjasjkdla998y1122";
	final byte[] bytes = new byte[1024];
	ThreadLocalRandom.current().nextBytes(bytes);
	final BigDecimal[] numbers = new BigDecimal[] {
		BigDecimal.valueOf(1), BigDecimal.valueOf(2), BigDecimal.valueOf(3)};
	final String[] strings = new String[] {"one", "two", "three"};

	final ObjectMapper objectMapper = new ObjectMapper();

	// Root
	ObjectNode root = objectMapper.createObjectNode();
	root.put("id", id.longValue());
	root.putNull("idOrNull");
	root.put("name", name);
	root.put("date", "1990-10-14");
	root.put("time", "12:12:43Z");
	root.put("timestamp", "1990-10-14T12:12:43Z");
	root.put("bytes", bytes);
	root.putArray("numbers").add(1).add(2).add(3);
	root.putArray("strings").add("one").add("two").add("three");
	root.putObject("nested").put("booleanField", true).put("decimalField", 12);

	final byte[] serializedJson = objectMapper.writeValueAsBytes(root);

	JsonRowDeserializationSchema deserializationSchema = new JsonRowDeserializationSchema.Builder(
		"{" +
		"    type: 'object'," +
		"    properties: {" +
		"         id: { type: 'integer' }," +
		"         idOrNull: { type: ['integer', 'null'] }," +
		"         name: { type: 'string' }," +
		"         date: { type: 'string', format: 'date' }," +
		"         time: { type: 'string', format: 'time' }," +
		"         timestamp: { type: 'string', format: 'date-time' }," +
		"         bytes: { type: 'string', contentEncoding: 'base64' }," +
		"         numbers: { type: 'array', items: { type: 'integer' } }," +
		"         strings: { type: 'array', items: { type: 'string' } }," +
		"         nested: { " +
		"             type: 'object'," +
		"             properties: { " +
		"                 booleanField: { type: 'boolean' }," +
		"                 decimalField: { type: 'number' }" +
		"             }" +
		"         }" +
		"    }" +
		"}").build();

	final Row expected = new Row(10);
	expected.setField(0, id);
	expected.setField(1, null);
	expected.setField(2, name);
	expected.setField(3, Date.valueOf("1990-10-14"));
	expected.setField(4, Time.valueOf("12:12:43"));
	expected.setField(5, Timestamp.valueOf("1990-10-14 12:12:43"));
	expected.setField(6, bytes);
	expected.setField(7, numbers);
	expected.setField(8, strings);
	final Row nestedRow = new Row(2);
	nestedRow.setField(0, true);
	nestedRow.setField(1, BigDecimal.valueOf(12));
	expected.setField(9, nestedRow);

	assertThat(serializedJson, whenDeserializedWith(deserializationSchema).equalsTo(expected));
}
 
Example 20
Source File: JsonRowDataSerDeSchemaTest.java    From flink with Apache License 2.0 4 votes vote down vote up
@Test
public void testSerDe() throws Exception {
	byte tinyint = 'c';
	short smallint = 128;
	int intValue = 45536;
	float floatValue = 33.333F;
	long bigint = 1238123899121L;
	String name = "asdlkjasjkdla998y1122";
	byte[] bytes = new byte[1024];
	ThreadLocalRandom.current().nextBytes(bytes);
	BigDecimal decimal = new BigDecimal("123.456789");
	Double[] doubles = new Double[]{1.1, 2.2, 3.3};
	LocalDate date = LocalDate.parse("1990-10-14");
	LocalTime time = LocalTime.parse("12:12:43");
	Timestamp timestamp3 = Timestamp.valueOf("1990-10-14 12:12:43.123");
	Timestamp timestamp9 = Timestamp.valueOf("1990-10-14 12:12:43.123456789");

	Map<String, Long> map = new HashMap<>();
	map.put("flink", 123L);

	Map<String, Map<String, Integer>> nestedMap = new HashMap<>();
	Map<String, Integer> innerMap = new HashMap<>();
	innerMap.put("key", 234);
	nestedMap.put("inner_map", innerMap);

	ObjectMapper objectMapper = new ObjectMapper();
	ArrayNode doubleNode = objectMapper.createArrayNode().add(1.1D).add(2.2D).add(3.3D);

	// Root
	ObjectNode root = objectMapper.createObjectNode();
	root.put("bool", true);
	root.put("tinyint", tinyint);
	root.put("smallint", smallint);
	root.put("int", intValue);
	root.put("bigint", bigint);
	root.put("float", floatValue);
	root.put("name", name);
	root.put("bytes", bytes);
	root.put("decimal", decimal);
	root.set("doubles", doubleNode);
	root.put("date", "1990-10-14");
	root.put("time", "12:12:43");
	root.put("timestamp3", "1990-10-14T12:12:43.123");
	root.put("timestamp9", "1990-10-14T12:12:43.123456789");
	root.putObject("map").put("flink", 123);
	root.putObject("map2map").putObject("inner_map").put("key", 234);

	byte[] serializedJson = objectMapper.writeValueAsBytes(root);

	DataType dataType = ROW(
		FIELD("bool", BOOLEAN()),
		FIELD("tinyint", TINYINT()),
		FIELD("smallint", SMALLINT()),
		FIELD("int", INT()),
		FIELD("bigint", BIGINT()),
		FIELD("float", FLOAT()),
		FIELD("name", STRING()),
		FIELD("bytes", BYTES()),
		FIELD("decimal", DECIMAL(9, 6)),
		FIELD("doubles", ARRAY(DOUBLE())),
		FIELD("date", DATE()),
		FIELD("time", TIME(0)),
		FIELD("timestamp3", TIMESTAMP(3)),
		FIELD("timestamp9", TIMESTAMP(9)),
		FIELD("map", MAP(STRING(), BIGINT())),
		FIELD("map2map", MAP(STRING(), MAP(STRING(), INT()))));
	RowType schema = (RowType) dataType.getLogicalType();
	RowDataTypeInfo resultTypeInfo = new RowDataTypeInfo(schema);

	JsonRowDataDeserializationSchema deserializationSchema = new JsonRowDataDeserializationSchema(
		schema, resultTypeInfo, false, false, TimestampFormat.ISO_8601);

	Row expected = new Row(16);
	expected.setField(0, true);
	expected.setField(1, tinyint);
	expected.setField(2, smallint);
	expected.setField(3, intValue);
	expected.setField(4, bigint);
	expected.setField(5, floatValue);
	expected.setField(6, name);
	expected.setField(7, bytes);
	expected.setField(8, decimal);
	expected.setField(9, doubles);
	expected.setField(10, date);
	expected.setField(11, time);
	expected.setField(12, timestamp3.toLocalDateTime());
	expected.setField(13, timestamp9.toLocalDateTime());
	expected.setField(14, map);
	expected.setField(15, nestedMap);

	RowData rowData = deserializationSchema.deserialize(serializedJson);
	Row actual = convertToExternal(rowData, dataType);
	assertEquals(expected, actual);

	// test serialization
	JsonRowDataSerializationSchema serializationSchema = new JsonRowDataSerializationSchema(schema,  TimestampFormat.ISO_8601);

	byte[] actualBytes = serializationSchema.serialize(rowData);
	assertEquals(new String(serializedJson), new String(actualBytes));
}