org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper Java Examples

@Test
public void getNumberOfTaskManagers() {
	try {
		String json = TestBaseUtils.getFromHTTP("http://localhost:" + getRestPort() + "/taskmanagers/");

		ObjectMapper mapper = new ObjectMapper();
		JsonNode response = mapper.readTree(json);
		ArrayNode taskManagers = (ArrayNode) response.get("taskmanagers");

		assertNotNull(taskManagers);
		assertEquals(NUM_TASK_MANAGERS, taskManagers.size());
	} catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 

/**
 * Converts a JSON schema into Flink's type information. Throws an exception if the schema
 * cannot converted because of loss of precision or too flexible schema.
 *
 * <p>The converter can resolve simple schema references to solve those cases where entities
 * are defined at the beginning and then used throughout a document.
 */
@SuppressWarnings("unchecked")
public static <T> TypeInformation<T> convert(String jsonSchema) {
	Preconditions.checkNotNull(jsonSchema, "JSON schema");
	final ObjectMapper mapper = new ObjectMapper();
	mapper.getFactory()
		.enable(JsonParser.Feature.ALLOW_COMMENTS)
		.enable(JsonParser.Feature.ALLOW_UNQUOTED_FIELD_NAMES)
		.enable(JsonParser.Feature.ALLOW_SINGLE_QUOTES);
	final JsonNode node;
	try {
		node = mapper.readTree(jsonSchema);
	} catch (IOException e) {
		throw new IllegalArgumentException(
			"Invalid JSON schema.", e);
	}
	return (TypeInformation<T>) convertType("<root>", node, node);
}
 

@Test
public void testDeserializeWithoutValue() throws Exception {
	ObjectMapper mapper = new ObjectMapper();
	ObjectNode initialKey = mapper.createObjectNode();
	initialKey.put("index", 4);
	byte[] serializedKey = mapper.writeValueAsBytes(initialKey);

	byte[] serializedValue = null;

	JSONKeyValueDeserializationSchema schema = new JSONKeyValueDeserializationSchema(false);
	ObjectNode deserializedValue = schema.deserialize(newConsumerRecord(serializedKey, serializedValue));

	Assert.assertTrue(deserializedValue.get("metadata") == null);
	Assert.assertEquals(4, deserializedValue.get("key").get("index").asInt());
	Assert.assertTrue(deserializedValue.get("value") == null);
}
 

@Test
public void testDeserializeWithoutKey() throws Exception {
	ObjectMapper mapper = new ObjectMapper();
	byte[] serializedKey = null;

	ObjectNode initialValue = mapper.createObjectNode();
	initialValue.put("word", "world");
	byte[] serializedValue = mapper.writeValueAsBytes(initialValue);

	JSONKeyValueDeserializationSchema schema = new JSONKeyValueDeserializationSchema(false);
	ObjectNode deserializedValue = schema.deserialize(newConsumerRecord(serializedKey, serializedValue));

	Assert.assertTrue(deserializedValue.get("metadata") == null);
	Assert.assertTrue(deserializedValue.get("key") == null);
	Assert.assertEquals("world", deserializedValue.get("value").get("word").asText());
}
 

@Test
public void testDeserializeWithoutKey() throws Exception {
	ObjectMapper mapper = new ObjectMapper();
	byte[] serializedKey = null;

	ObjectNode initialValue = mapper.createObjectNode();
	initialValue.put("word", "world");
	byte[] serializedValue = mapper.writeValueAsBytes(initialValue);

	JSONKeyValueDeserializationSchema schema = new JSONKeyValueDeserializationSchema(false);
	ObjectNode deserializedValue = schema.deserialize(newConsumerRecord(serializedKey, serializedValue));

	Assert.assertTrue(deserializedValue.get("metadata") == null);
	Assert.assertTrue(deserializedValue.get("key") == null);
	Assert.assertEquals("world", deserializedValue.get("value").get("word").asText());
}
 

/**
 * Tests that we can marshal and unmarshal the response.
 */
@Test
public void testJsonMarshalling() throws Exception {
	final R expected = getTestResponseInstance();

	ObjectMapper objectMapper = RestMapperUtils.getStrictObjectMapper();
	final String marshalled = objectMapper.writeValueAsString(expected);

	final Collection<Class<?>> typeParameters = getTypeParameters();
	final JavaType type;

	if (typeParameters.isEmpty()) {
		type = objectMapper.getTypeFactory().constructType(getTestResponseClass());
	} else {
		type = objectMapper.getTypeFactory().constructParametricType(
			getTestResponseClass(),
			typeParameters.toArray(new Class<?>[typeParameters.size()]));
	}

	final R unmarshalled = objectMapper.readValue(marshalled, type);
	assertOriginalEqualsToUnmarshalled(expected, unmarshalled);
}
 

@Test
public void testDeserializeWithoutValue() throws Exception {
	ObjectMapper mapper = new ObjectMapper();
	ObjectNode initialKey = mapper.createObjectNode();
	initialKey.put("index", 4);
	byte[] serializedKey = mapper.writeValueAsBytes(initialKey);

	byte[] serializedValue = null;

	JSONKeyValueDeserializationSchema schema = new JSONKeyValueDeserializationSchema(false);
	ObjectNode deserializedValue = schema.deserialize(newConsumerRecord(serializedKey, serializedValue));

	Assert.assertTrue(deserializedValue.get("metadata") == null);
	Assert.assertEquals(4, deserializedValue.get("key").get("index").asInt());
	Assert.assertTrue(deserializedValue.get("value") == null);
}
 

@Test
public void testSerDeSQLTimestampFormat() throws Exception{
	RowType rowType = (RowType) ROW(
		FIELD("timestamp3", TIMESTAMP(3)),
		FIELD("timestamp9", TIMESTAMP(9))
	).getLogicalType();

	JsonRowDataDeserializationSchema deserializationSchema = new JsonRowDataDeserializationSchema(
		rowType, new RowDataTypeInfo(rowType), false, false, TimestampFormat.SQL);
	JsonRowDataSerializationSchema serializationSchema = new JsonRowDataSerializationSchema(rowType, TimestampFormat.SQL);

	ObjectMapper objectMapper = new ObjectMapper();

	ObjectNode root = objectMapper.createObjectNode();
	root.put("timestamp3", "1990-10-14 12:12:43.123");
	root.put("timestamp9", "1990-10-14 12:12:43.123456789");
	byte[] serializedJson = objectMapper.writeValueAsBytes(root);
	RowData rowData = deserializationSchema.deserialize(serializedJson);
	byte[] actual = serializationSchema.serialize(rowData);
	assertEquals(new String(serializedJson), new String(actual));
}
 

private Object convertField(
	ObjectMapper mapper,
	DeserializationRuntimeConverter fieldConverter,
	String fieldName,
	JsonNode field) {
	if (field == null) {
		if (failOnMissingField) {
			throw new IllegalStateException(
				"Could not find field with name '" + fieldName + "'.");
		} else {
			return null;
		}
	} else {
		return fieldConverter.convert(mapper, field);
	}
}
 

/**
 * Tests that we can marshal and unmarshal the response.
 */
@Test
public void testJsonMarshalling() throws Exception {
	final R expected = getTestResponseInstance();

	ObjectMapper objectMapper = RestMapperUtils.getStrictObjectMapper();
	final String marshalled = objectMapper.writeValueAsString(expected);

	final Collection<Class<?>> typeParameters = getTypeParameters();
	final JavaType type;

	if (typeParameters.isEmpty()) {
		type = objectMapper.getTypeFactory().constructType(getTestResponseClass());
	} else {
		type = objectMapper.getTypeFactory().constructParametricType(
			getTestResponseClass(),
			typeParameters.toArray(new Class<?>[typeParameters.size()]));
	}

	final R unmarshalled = objectMapper.readValue(marshalled, type);
	assertOriginalEqualsToUnmarshalled(expected, unmarshalled);
}
 

private LocalDateTime convertToLocalDateTime(ObjectMapper mapper, JsonNode jsonNode) {
	// according to RFC 3339 every date-time must have a timezone;
	// until we have full timezone support, we only support UTC;
	// users can parse their time as string as a workaround
	TemporalAccessor parsedTimestamp = RFC3339_TIMESTAMP_FORMAT.parse(jsonNode.asText());

	ZoneOffset zoneOffset = parsedTimestamp.query(TemporalQueries.offset());

	if (zoneOffset != null && zoneOffset.getTotalSeconds() != 0) {
		throw new IllegalStateException(
			"Invalid timestamp format. Only a timestamp in UTC timezone is supported yet. " +
				"Format: yyyy-MM-dd'T'HH:mm:ss.SSS'Z'");
	}

	LocalTime localTime = parsedTimestamp.query(TemporalQueries.localTime());
	LocalDate localDate = parsedTimestamp.query(TemporalQueries.localDate());

	return LocalDateTime.of(localDate, localTime);
}
 

/**
 * Tests that we can marshal and unmarshal JobDetails instances.
 */
@Test
public void testJobDetailsMarshalling() throws JsonProcessingException {
	final JobDetails expected = new JobDetails(
		new JobID(),
		"foobar",
		1L,
		10L,
		9L,
		JobStatus.RUNNING,
		8L,
		new int[]{1, 3, 3, 7, 4, 2, 7, 3, 3},
		42);

	final ObjectMapper objectMapper = RestMapperUtils.getStrictObjectMapper();

	final JsonNode marshalled = objectMapper.valueToTree(expected);

	final JobDetails unmarshalled = objectMapper.treeToValue(marshalled, JobDetails.class);

	assertEquals(expected, unmarshalled);
}
 

@Test
public void testDeserializeWithMetadata() throws Exception {
	ObjectMapper mapper = new ObjectMapper();
	ObjectNode initialKey = mapper.createObjectNode();
	initialKey.put("index", 4);
	byte[] serializedKey = mapper.writeValueAsBytes(initialKey);

	ObjectNode initialValue = mapper.createObjectNode();
	initialValue.put("word", "world");
	byte[] serializedValue = mapper.writeValueAsBytes(initialValue);

	JSONKeyValueDeserializationSchema schema = new JSONKeyValueDeserializationSchema(true);
	final ConsumerRecord<byte[], byte[]> consumerRecord =
			newConsumerRecord("topic#1", 3, 4L, serializedKey, serializedValue);
	ObjectNode deserializedValue = schema.deserialize(consumerRecord);

	Assert.assertEquals(4, deserializedValue.get("key").get("index").asInt());
	Assert.assertEquals("world", deserializedValue.get("value").get("word").asText());
	Assert.assertEquals("topic#1", deserializedValue.get("metadata").get("topic").asText());
	Assert.assertEquals(4, deserializedValue.get("metadata").get("offset").asInt());
	Assert.assertEquals(3, deserializedValue.get("metadata").get("partition").asInt());
}
 

public static Map<String, String> fromKeyValueJsonArray(String jsonString) {
	try {
		Map<String, String> map = new HashMap<>();
		ObjectMapper m = new ObjectMapper();
		ArrayNode array = (ArrayNode) m.readTree(jsonString);

		Iterator<JsonNode> elements = array.elements();
		while (elements.hasNext()) {
			JsonNode node = elements.next();
			String key = node.get("key").asText();
			String value = node.get("value").asText();
			map.put(key, value);
		}

		return map;
	}
	catch (Exception e) {
		throw new RuntimeException(e.getMessage(), e);
	}
}
 

public static Map<String, String> fromKeyValueJsonArray(String jsonString) {
	try {
		Map<String, String> map = new HashMap<>();
		ObjectMapper m = new ObjectMapper();
		ArrayNode array = (ArrayNode) m.readTree(jsonString);

		Iterator<JsonNode> elements = array.elements();
		while (elements.hasNext()) {
			JsonNode node = elements.next();
			String key = node.get("key").asText();
			String value = node.get("value").asText();
			map.put(key, value);
		}

		return map;
	}
	catch (Exception e) {
		throw new RuntimeException(e.getMessage(), e);
	}
}
 

@Test
public void getTaskManagerLogAndStdoutFiles() {
	try {
		String json = TestBaseUtils.getFromHTTP("http://localhost:" + getRestPort() + "/taskmanagers/");

		ObjectMapper mapper = new ObjectMapper();
		JsonNode parsed = mapper.readTree(json);
		ArrayNode taskManagers = (ArrayNode) parsed.get("taskmanagers");
		JsonNode taskManager = taskManagers.get(0);
		String id = taskManager.get("id").asText();

		WebMonitorUtils.LogFileLocation logFiles = WebMonitorUtils.LogFileLocation.find(CLUSTER_CONFIGURATION);

		//we check for job manager log files, since no separate taskmanager logs exist
		FileUtils.writeStringToFile(logFiles.logFile, "job manager log");
		String logs = TestBaseUtils.getFromHTTP("http://localhost:" + getRestPort() + "/taskmanagers/" + id + "/log");
		assertTrue(logs.contains("job manager log"));

		FileUtils.writeStringToFile(logFiles.stdOutFile, "job manager out");
		logs = TestBaseUtils.getFromHTTP("http://localhost:" + getRestPort() + "/taskmanagers/" + id + "/stdout");
		assertTrue(logs.contains("job manager out"));
	} catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 

@Test
public void getTaskmanagers() throws Exception {
	String json = TestBaseUtils.getFromHTTP("http://localhost:" + getRestPort() + "/taskmanagers/");

	ObjectMapper mapper = new ObjectMapper();
	JsonNode parsed = mapper.readTree(json);
	ArrayNode taskManagers = (ArrayNode) parsed.get("taskmanagers");

	assertNotNull(taskManagers);
	assertEquals(NUM_TASK_MANAGERS, taskManagers.size());

	JsonNode taskManager = taskManagers.get(0);
	assertNotNull(taskManager);
	assertEquals(NUM_SLOTS, taskManager.get("slotsNumber").asInt());
	assertTrue(taskManager.get("freeSlots").asInt() <= NUM_SLOTS);
}
 

@Test
public void getNumberOfTaskManagers() {
	try {
		String json = TestBaseUtils.getFromHTTP("http://localhost:" + getRestPort() + "/taskmanagers/");

		ObjectMapper mapper = new ObjectMapper();
		JsonNode response = mapper.readTree(json);
		ArrayNode taskManagers = (ArrayNode) response.get("taskmanagers");

		assertNotNull(taskManagers);
		assertEquals(NUM_TASK_MANAGERS, taskManagers.size());
	} catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 

/**
 * Converts a JSON schema into Flink's type information. Throws an exception if the schema
 * cannot converted because of loss of precision or too flexible schema.
 *
 * <p>The converter can resolve simple schema references to solve those cases where entities
 * are defined at the beginning and then used throughout a document.
 */
@SuppressWarnings("unchecked")
public static <T> TypeInformation<T> convert(String jsonSchema) {
	Preconditions.checkNotNull(jsonSchema, "JSON schema");
	final ObjectMapper mapper = new ObjectMapper();
	mapper.getFactory()
		.enable(JsonParser.Feature.ALLOW_COMMENTS)
		.enable(JsonParser.Feature.ALLOW_UNQUOTED_FIELD_NAMES)
		.enable(JsonParser.Feature.ALLOW_SINGLE_QUOTES);
	final JsonNode node;
	try {
		node = mapper.readTree(jsonSchema);
	} catch (IOException e) {
		throw new IllegalArgumentException(
			"Invalid JSON schema.", e);
	}
	return (TypeInformation<T>) convertType("<root>", node, node);
}
 

@Test
public void getTaskManagers() throws Exception {
	String json = TestBaseUtils.getFromHTTP("http://localhost:" + getRestPort() + "/taskmanagers/");

	ObjectMapper mapper = new ObjectMapper();
	JsonNode parsed = mapper.readTree(json);
	ArrayNode taskManagers = (ArrayNode) parsed.get("taskmanagers");

	assertNotNull(taskManagers);
	assertEquals(NUM_TASK_MANAGERS, taskManagers.size());

	JsonNode taskManager = taskManagers.get(0);
	assertNotNull(taskManager);
	assertEquals(NUM_SLOTS, taskManager.get("slotsNumber").asInt());
	assertTrue(taskManager.get("freeSlots").asInt() <= NUM_SLOTS);
}
 

@Test
public void getTaskManagerLogAndStdoutFiles() {
	try {
		String json = TestBaseUtils.getFromHTTP("http://localhost:" + getRestPort() + "/taskmanagers/");

		ObjectMapper mapper = new ObjectMapper();
		JsonNode parsed = mapper.readTree(json);
		ArrayNode taskManagers = (ArrayNode) parsed.get("taskmanagers");
		JsonNode taskManager = taskManagers.get(0);
		String id = taskManager.get("id").asText();

		WebMonitorUtils.LogFileLocation logFiles = WebMonitorUtils.LogFileLocation.find(CLUSTER_CONFIGURATION);

		//we check for job manager log files, since no separate taskmanager logs exist
		FileUtils.writeStringToFile(logFiles.logFile, "job manager log");
		String logs = TestBaseUtils.getFromHTTP("http://localhost:" + getRestPort() + "/taskmanagers/" + id + "/log");
		assertTrue(logs.contains("job manager log"));

		FileUtils.writeStringToFile(logFiles.stdOutFile, "job manager out");
		logs = TestBaseUtils.getFromHTTP("http://localhost:" + getRestPort() + "/taskmanagers/" + id + "/stdout");
		assertTrue(logs.contains("job manager out"));
	} catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 

@Test
public void getTaskmanagers() throws Exception {
	String json = TestBaseUtils.getFromHTTP("http://localhost:" + getRestPort() + "/taskmanagers/");

	ObjectMapper mapper = new ObjectMapper();
	JsonNode parsed = mapper.readTree(json);
	ArrayNode taskManagers = (ArrayNode) parsed.get("taskmanagers");

	assertNotNull(taskManagers);
	assertEquals(NUM_TASK_MANAGERS, taskManagers.size());

	JsonNode taskManager = taskManagers.get(0);
	assertNotNull(taskManager);
	assertEquals(NUM_SLOTS, taskManager.get("slotsNumber").asInt());
	assertTrue(taskManager.get("freeSlots").asInt() <= NUM_SLOTS);
}
 

private LocalTime convertToLocalTime(ObjectMapper mapper, JsonNode jsonNode) {
	// according to RFC 3339 every full-time must have a timezone;
	// until we have full timezone support, we only support UTC;
	// users can parse their time as string as a workaround

	TemporalAccessor parsedTime = RFC3339_TIME_FORMAT.parse(jsonNode.asText());

	ZoneOffset zoneOffset = parsedTime.query(TemporalQueries.offset());
	LocalTime localTime = parsedTime.query(TemporalQueries.localTime());

	if (zoneOffset != null && zoneOffset.getTotalSeconds() != 0 || localTime.getNano() != 0) {
		throw new IllegalStateException(
			"Invalid time format. Only a time in UTC timezone without milliseconds is supported yet.");
	}

	return localTime;
}
 

@Test
public void testDeserializeWithMetadata() throws Exception {
	ObjectMapper mapper = new ObjectMapper();
	ObjectNode initialKey = mapper.createObjectNode();
	initialKey.put("index", 4);
	byte[] serializedKey = mapper.writeValueAsBytes(initialKey);

	ObjectNode initialValue = mapper.createObjectNode();
	initialValue.put("word", "world");
	byte[] serializedValue = mapper.writeValueAsBytes(initialValue);

	JSONKeyValueDeserializationSchema schema = new JSONKeyValueDeserializationSchema(true);
	final ConsumerRecord<byte[], byte[]> consumerRecord =
			newConsumerRecord("topic#1", 3, 4L, serializedKey, serializedValue);
	ObjectNode deserializedValue = schema.deserialize(consumerRecord);

	Assert.assertEquals(4, deserializedValue.get("key").get("index").asInt());
	Assert.assertEquals("world", deserializedValue.get("value").get("word").asText());
	Assert.assertEquals("topic#1", deserializedValue.get("metadata").get("topic").asText());
	Assert.assertEquals(4, deserializedValue.get("metadata").get("offset").asInt());
	Assert.assertEquals(3, deserializedValue.get("metadata").get("partition").asInt());
}
 

@Test
public void testDeserializeWithoutValue() throws Exception {
	ObjectMapper mapper = new ObjectMapper();
	ObjectNode initialKey = mapper.createObjectNode();
	initialKey.put("index", 4);
	byte[] serializedKey = mapper.writeValueAsBytes(initialKey);

	byte[] serializedValue = null;

	JSONKeyValueDeserializationSchema schema = new JSONKeyValueDeserializationSchema(false);
	ObjectNode deserializedValue = schema.deserialize(newConsumerRecord(serializedKey, serializedValue));

	Assert.assertTrue(deserializedValue.get("metadata") == null);
	Assert.assertEquals(4, deserializedValue.get("key").get("index").asInt());
	Assert.assertTrue(deserializedValue.get("value") == null);
}
 

@Test
public void testHistoryServerIntegration() throws Exception {
	final int numJobs = 2;
	for (int x = 0; x < numJobs; x++) {
		runJob();
	}
	createLegacyArchive(jmDirectory.toPath());

	CountDownLatch numFinishedPolls = new CountDownLatch(1);

	Configuration historyServerConfig = new Configuration();
	historyServerConfig.setString(HistoryServerOptions.HISTORY_SERVER_ARCHIVE_DIRS, jmDirectory.toURI().toString());
	historyServerConfig.setString(HistoryServerOptions.HISTORY_SERVER_WEB_DIR, hsDirectory.getAbsolutePath());

	historyServerConfig.setInteger(HistoryServerOptions.HISTORY_SERVER_WEB_PORT, 0);

	// the job is archived asynchronously after env.execute() returns
	File[] archives = jmDirectory.listFiles();
	while (archives == null || archives.length != numJobs + 1) {
		Thread.sleep(50);
		archives = jmDirectory.listFiles();
	}

	HistoryServer hs = new HistoryServer(historyServerConfig, numFinishedPolls);
	try {
		hs.start();
		String baseUrl = "http://localhost:" + hs.getWebPort();
		numFinishedPolls.await(10L, TimeUnit.SECONDS);

		ObjectMapper mapper = new ObjectMapper();
		String response = getFromHTTP(baseUrl + JobsOverviewHeaders.URL);
		MultipleJobsDetails overview = mapper.readValue(response, MultipleJobsDetails.class);

		Assert.assertEquals(numJobs + 1, overview.getJobs().size());
	} finally {
		hs.stop();
	}
}
 

public ControlMessage(ControlEvent event) {
    this.type = event.getClass().getName();
    ObjectMapper mapper = new ObjectMapper();
    try {
        this.payload = mapper.writeValueAsString(event);
    } catch (JsonProcessingException e) {
        throw new IllegalArgumentException(e);
    }
}
 

/**
 * Tests that we can marshal and unmarshal the response.
 */
@Test
public void testJsonMarshalling() throws Exception {
	final R expected = getTestRequestInstance();

	ObjectMapper objectMapper = RestMapperUtils.getStrictObjectMapper();
	final String marshalled = objectMapper.writeValueAsString(expected);

	final R unmarshalled = objectMapper.readValue(marshalled, getTestRequestClass());
	assertOriginalEqualsToUnmarshalled(expected, unmarshalled);
}
 

private static JsonNode fromPath(String path) {
  URL moduleUrl = ProtobufKafkaSourceProvider.class.getClassLoader().getResource(path);
  ObjectMapper mapper = new ObjectMapper(new YAMLFactory());
  try {
    return mapper.readTree(moduleUrl);
  } catch (IOException e) {
    throw new RuntimeException(e);
  }
}
 

/**
 * Read a {@code StatefulFunction} module definition.
 *
 * <p>A valid resource module definition has to contain the metadata associated with this module,
 * such as its type.
 */
private static JsonNode readAndValidateModuleTree(ObjectMapper mapper, URL moduleYamlFile)
    throws IOException {
  JsonNode root = mapper.readTree(moduleYamlFile);
  validateMeta(moduleYamlFile, root);
  return root;
}