org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.JsonNode Java Examples

@Test
public void exampleUsage() {
  JsonNode ingressDefinition =
      loadAsJsonFromClassResource(
          getClass().getClassLoader(), "routable-protobuf-kinesis-ingress.yaml");
  JsonIngressSpec<?> spec =
      new JsonIngressSpec<>(
          PolyglotKinesisIOTypes.ROUTABLE_PROTOBUF_KINESIS_INGRESS_TYPE,
          new IngressIdentifier<>(Message.class, "foo", "bar"),
          ingressDefinition);

  RoutableProtobufKinesisSourceProvider provider = new RoutableProtobufKinesisSourceProvider();
  SourceFunction<?> source = provider.forSpec(spec);

  assertThat(source, instanceOf(FlinkKinesisConsumer.class));
}
 

@Test
public void getTaskmanagers() throws Exception {
	String json = TestBaseUtils.getFromHTTP("http://localhost:" + getRestPort() + "/taskmanagers/");

	ObjectMapper mapper = new ObjectMapper();
	JsonNode parsed = mapper.readTree(json);
	ArrayNode taskManagers = (ArrayNode) parsed.get("taskmanagers");

	assertNotNull(taskManagers);
	assertEquals(NUM_TASK_MANAGERS, taskManagers.size());

	JsonNode taskManager = taskManagers.get(0);
	assertNotNull(taskManager);
	assertEquals(NUM_SLOTS, taskManager.get("slotsNumber").asInt());
	assertTrue(taskManager.get("freeSlots").asInt() <= NUM_SLOTS);
}
 

private RestAPISnapshot createSnapshot(final DocumentingRestEndpoint restEndpoint) {
	final List<JsonNode> calls = restEndpoint.getSpecs().stream()
		// we only compare compatibility within the given version
		.filter(spec -> spec.getSupportedAPIVersions().contains(apiVersion))
		.map(spec -> {
			final ObjectNode json = OBJECT_MAPPER.createObjectNode();

			for (final CompatibilityRoutine<?> routine : CompatibilityRoutines.ROUTINES) {
				final Object extract = routine.getContainer(spec);
				json.set(routine.getKey(), OBJECT_MAPPER.valueToTree(extract));
			}

			return json;
		})
		.collect(Collectors.toList());

	return new RestAPISnapshot(calls);
}
 

public static Map<String, String> propertiesAt(JsonNode node, JsonPointer pointer) {
  node = node.at(pointer);
  if (node.isMissingNode()) {
    return Collections.emptyMap();
  }
  if (!node.isArray()) {
    throw new WrongTypeException(pointer, "not a key-value list");
  }
  Map<String, String> properties = new LinkedHashMap<>();
  for (JsonNode listElement : node) {
    Iterator<Map.Entry<String, JsonNode>> fields = listElement.fields();
    if (!fields.hasNext()) {
      throw new WrongTypeException(pointer, "not a key-value list");
    }
    Map.Entry<String, JsonNode> field = fields.next();
    properties.put(field.getKey(), field.getValue().asText());
  }
  return properties;
}
 

private static TypeInformation<?> convertStringFormat(String location, JsonNode node) {
	if (!node.isTextual()) {
		throw new IllegalArgumentException("Invalid '" + FORMAT + "' property in node: " + location);
	}

	switch (node.asText()) {
		case FORMAT_DATE:
			return Types.SQL_DATE;
		case FORMAT_TIME:
			return Types.SQL_TIME;
		case FORMAT_DATE_TIME:
			return Types.SQL_TIMESTAMP;
		default:
			return Types.STRING; // unlikely that we will support other formats in the future
	}
}
 

@Override
public JobConfigInfo deserialize(
		JsonParser jsonParser,
		DeserializationContext deserializationContext) throws IOException {
	JsonNode rootNode = jsonParser.readValueAsTree();

	final JobID jobId = JobID.fromHexString(rootNode.get(FIELD_NAME_JOB_ID).asText());
	final String jobName = rootNode.get(FIELD_NAME_JOB_NAME).asText();

	final ExecutionConfigInfo executionConfigInfo;

	if (rootNode.has(FIELD_NAME_EXECUTION_CONFIG)) {
		executionConfigInfo = RestMapperUtils.getStrictObjectMapper().treeToValue(rootNode.get(FIELD_NAME_EXECUTION_CONFIG), ExecutionConfigInfo.class);
	} else {
		executionConfigInfo = null;
	}

	return new JobConfigInfo(jobId, jobName, executionConfigInfo);
}
 

public static Map<String, String> fromKeyValueJsonArray(String jsonString) {
	try {
		Map<String, String> map = new HashMap<>();
		ObjectMapper m = new ObjectMapper();
		ArrayNode array = (ArrayNode) m.readTree(jsonString);

		Iterator<JsonNode> elements = array.elements();
		while (elements.hasNext()) {
			JsonNode node = elements.next();
			String key = node.get("key").asText();
			String value = node.get("value").asText();
			map.put(key, value);
		}

		return map;
	}
	catch (Exception e) {
		throw new RuntimeException(e.getMessage(), e);
	}
}
 

private static JsonNode resolveReference(String ref, JsonNode origin, JsonNode root) {
	if (!ref.startsWith("#")) {
		throw new IllegalArgumentException("Only JSON schemes with simple references " +
			"(one indirection in the same document) are supported yet. But was: " + ref);
	}
	final String path = ref.substring(1);
	final JsonNode foundNode = root.at(path);
	if (foundNode.isMissingNode()) {
		throw new IllegalArgumentException("Could not find reference: " + ref);
	}
	// prevent obvious cyclic references
	if (foundNode == origin) {
		throw new IllegalArgumentException("Cyclic references are not supported:" + ref);
	}
	return foundNode;
}
 

/**
 * Tests that we can marshal and unmarshal JobDetails instances.
 */
@Test
public void testJobDetailsMarshalling() throws JsonProcessingException {
	final JobDetails expected = new JobDetails(
		new JobID(),
		"foobar",
		1L,
		10L,
		9L,
		JobStatus.RUNNING,
		8L,
		new int[]{1, 3, 3, 7, 4, 2, 7, 3, 3},
		42);

	final ObjectMapper objectMapper = RestMapperUtils.getStrictObjectMapper();

	final JsonNode marshalled = objectMapper.valueToTree(expected);

	final JobDetails unmarshalled = objectMapper.treeToValue(marshalled, JobDetails.class);

	assertEquals(expected, unmarshalled);
}
 

private static URI functionUri(JsonNode functionNode) {
  String uri = Selectors.textAt(functionNode, SpecPointers.ENDPOINT);
  URI typedUri = URI.create(uri);
  @Nullable String scheme = typedUri.getScheme();
  if (scheme == null) {
    throw new IllegalArgumentException(
        "Missing scheme in function endpoint "
            + uri
            + "; an http or https scheme must be provided.");
  }
  if (scheme.equalsIgnoreCase("http")
      || scheme.equalsIgnoreCase("https")
      || scheme.equalsIgnoreCase("http+unix")
      || scheme.equalsIgnoreCase("https+unix")) {
    return typedUri;
  }
  throw new IllegalArgumentException(
      "Missing scheme in function endpoint "
          + uri
          + "; an http or https or http+unix or https+unix scheme must be provided.");
}
 

private static TypeInformation<?> convertStringFormat(String location, JsonNode node) {
	if (!node.isTextual()) {
		throw new IllegalArgumentException("Invalid '" + FORMAT + "' property in node: " + location);
	}

	switch (node.asText()) {
		case FORMAT_DATE:
			return Types.SQL_DATE;
		case FORMAT_TIME:
			return Types.SQL_TIME;
		case FORMAT_DATE_TIME:
			return Types.SQL_TIMESTAMP;
		default:
			return Types.STRING; // unlikely that we will support other formats in the future
	}
}
 

private static JsonNode resolveReference(String ref, JsonNode origin, JsonNode root) {
	if (!ref.startsWith("#")) {
		throw new IllegalArgumentException("Only JSON schemes with simple references " +
			"(one indirection in the same document) are supported yet. But was: " + ref);
	}
	final String path = ref.substring(1);
	final JsonNode foundNode = root.at(path);
	if (foundNode.isMissingNode()) {
		throw new IllegalArgumentException("Could not find reference: " + ref);
	}
	// prevent obvious cyclic references
	if (foundNode == origin) {
		throw new IllegalArgumentException("Cyclic references are not supported:" + ref);
	}
	return foundNode;
}
 

private static List<StateSpec> functionStateSpecParserV2(JsonNode functionNode) {
  final Iterable<? extends JsonNode> stateSpecNodes =
      Selectors.listAt(functionNode, SpecPointers.STATES);
  final List<StateSpec> stateSpecs = new ArrayList<>();

  stateSpecNodes.forEach(
      stateSpecNode -> {
        final String name = Selectors.textAt(stateSpecNode, StateSpecPointers.NAME);
        final Optional<Duration> optionalStateExpireDuration =
            optionalStateExpireDuration(stateSpecNode);
        if (optionalStateExpireDuration.isPresent()) {
          stateSpecs.add(new StateSpec(name, optionalStateExpireDuration.get()));
        } else {
          stateSpecs.add(new StateSpec(name));
        }
      });
  return stateSpecs;
}
 

private static TypeInformation<?> convertStringEncoding(String location, JsonNode node) {
	if (!node.isTextual()) {
		throw new IllegalArgumentException("Invalid '" + CONTENT_ENCODING + "' property in node: " + location);
	}

	// "If the instance value is a string, this property defines that the string SHOULD
	// be interpreted as binary data and decoded using the encoding named by this property."

	switch (node.asText()) {
		case CONTENT_ENCODING_BASE64:
			return Types.PRIMITIVE_ARRAY(Types.BYTE);
		default:
			// we fail hard here:
			// this gives us the chance to support more encodings in the future without problems
			// of backwards compatibility
			throw new IllegalArgumentException("Invalid encoding '" + node.asText() + "' in node: " + location);
	}
}
 

private static TypeInformation<?> convertStringEncoding(String location, JsonNode node) {
	if (!node.isTextual()) {
		throw new IllegalArgumentException("Invalid '" + CONTENT_ENCODING + "' property in node: " + location);
	}

	// "If the instance value is a string, this property defines that the string SHOULD
	// be interpreted as binary data and decoded using the encoding named by this property."

	switch (node.asText()) {
		case CONTENT_ENCODING_BASE64:
			return Types.PRIMITIVE_ARRAY(Types.BYTE);
		default:
			// we fail hard here:
			// this gives us the chance to support more encodings in the future without problems
			// of backwards compatibility
			throw new IllegalArgumentException("Invalid encoding '" + node.asText() + "' in node: " + location);
	}
}
 

private static Router<Message> dynamicRouter(JsonNode router) {
  String addressTemplate = Selectors.textAt(router, Pointers.Routers.SPEC_TARGET);
  String descriptorSetPath = Selectors.textAt(router, Pointers.Routers.SPEC_DESCRIPTOR);
  String messageType = Selectors.textAt(router, Pointers.Routers.SPEC_MESSAGE_TYPE);

  ProtobufDescriptorMap descriptorPath = protobufDescriptorMap(descriptorSetPath);
  Optional<Descriptors.GenericDescriptor> maybeDescriptor =
      descriptorPath.getDescriptorByName(messageType);
  if (!maybeDescriptor.isPresent()) {
    throw new IllegalStateException(
        "Error while processing a router definition. Unable to locate a message "
            + messageType
            + " in a descriptor set "
            + descriptorSetPath);
  }
  return ProtobufRouter.forAddressTemplate(
      (Descriptors.Descriptor) maybeDescriptor.get(), addressTemplate);
}
 

static Optional<KafkaProducerSemantic> optionalDeliverySemantic(JsonNode json) {
  if (json.at(DELIVERY_SEMANTICS_POINTER).isMissingNode()) {
    return Optional.empty();
  }

  String deliverySemanticType =
      Selectors.textAt(json, DELIVERY_SEMANTICS_TYPE_POINTER).toLowerCase(Locale.ENGLISH);
  switch (deliverySemanticType) {
    case "at-least-once":
      return Optional.of(KafkaProducerSemantic.AT_LEAST_ONCE);
    case "exactly-once":
      return Optional.of(KafkaProducerSemantic.EXACTLY_ONCE);
    case "none":
      return Optional.of(KafkaProducerSemantic.NONE);
    default:
      throw new IllegalArgumentException(
          "Invalid delivery semantic type: "
              + deliverySemanticType
              + "; valid types are [at-least-once, exactly-once, none]");
  }
}
 

@Override
public RowData nextRecord(RowData reuse) throws IOException {
	GenericRowData csvRow = null;
	while (csvRow == null) {
		try {
			JsonNode root = iterator.nextValue();
			csvRow = (GenericRowData) runtimeConverter.convert(root);
		} catch (NoSuchElementException e) {
			end = true;
			return null;
		} catch (Throwable t) {
			if (!ignoreParseErrors) {
				throw new IOException("Failed to deserialize CSV row.", t);
			}
		}
	}

	GenericRowData returnRecord = rowData;
	for (int i = 0; i < csvSelectFieldToCsvFieldMapping.length; i++) {
		returnRecord.setField(csvSelectFieldToProjectFieldMapping[i],
			csvRow.getField(csvSelectFieldToCsvFieldMapping[i]));
	}
	emitted++;
	return returnRecord;
}
 

/**
 * Reads the given archive file and returns a {@link Collection} of contained {@link ArchivedJson}.
 *
 * @param file archive to extract
 * @return collection of archived jsons
 * @throws IOException if the file can't be opened, read or doesn't contain valid json
 */
public static Collection<ArchivedJson> getArchivedJsons(Path file) throws IOException {
	try (FSDataInputStream input = file.getFileSystem().open(file);
		ByteArrayOutputStream output = new ByteArrayOutputStream()) {
		IOUtils.copyBytes(input, output);

		JsonNode archive = mapper.readTree(output.toByteArray());

		Collection<ArchivedJson> archives = new ArrayList<>();
		for (JsonNode archivePart : archive.get(ARCHIVE)) {
			String path = archivePart.get(PATH).asText();
			String json = archivePart.get(JSON).asText();
			archives.add(new ArchivedJson(path, json));
		}
		return archives;
	}
}
 

@Test
public void exampleUsage() {
  JsonNode ingressDefinition =
      loadAsJsonFromClassResource(
          getClass().getClassLoader(), "routable-protobuf-kafka-ingress.yaml");
  JsonIngressSpec<?> spec =
      new JsonIngressSpec<>(
          ProtobufKafkaIngressTypes.ROUTABLE_PROTOBUF_KAFKA_INGRESS_TYPE,
          new IngressIdentifier<>(Message.class, "foo", "bar"),
          ingressDefinition);

  RoutableProtobufKafkaSourceProvider provider = new RoutableProtobufKafkaSourceProvider();
  SourceFunction<?> source = provider.forSpec(spec);

  assertThat(source, instanceOf(FlinkKafkaConsumer.class));
}
 

public static Optional<String> optionalTextAt(JsonNode node, JsonPointer pointer) {
  node = node.at(pointer);
  if (node.isMissingNode()) {
    return Optional.empty();
  }
  if (!node.isTextual()) {
    throw new WrongTypeException(pointer, "not a string");
  }
  return Optional.of(node.asText());
}
 

@Test
public void exampleUsage() {
  JsonNode egressDefinition =
      loadAsJsonFromClassResource(getClass().getClassLoader(), "generic-kafka-egress.yaml");
  JsonEgressSpec<?> spec =
      new JsonEgressSpec<>(
          KafkaEgressTypes.GENERIC_KAFKA_EGRESS_TYPE,
          new EgressIdentifier<>("foo", "bar", Any.class),
          egressDefinition);

  GenericKafkaSinkProvider provider = new GenericKafkaSinkProvider();
  SinkFunction<?> sink = provider.forSpec(spec);

  assertThat(sink, instanceOf(FlinkKafkaProducer.class));
}
 

private static List<TargetFunctionType> parseRoutableTargetFunctionTypes(
    JsonNode routableStreamNode) {
  final List<TargetFunctionType> targets = new ArrayList<>();
  for (String namespaceAndName :
      Selectors.textListAt(routableStreamNode, Streams.TARGETS_POINTER)) {
    NamespaceNamePair namespaceNamePair = NamespaceNamePair.from(namespaceAndName);
    targets.add(
        TargetFunctionType.newBuilder()
            .setNamespace(namespaceNamePair.namespace())
            .setType(namespaceNamePair.name())
            .build());
  }
  return targets;
}
 

private static <X> X jsonToObject(final JsonNode jsonContainer, final Class<X> containerClass) {
	try {
		return OBJECT_MAPPER.treeToValue(jsonContainer, containerClass);
	} catch (JsonProcessingException e) {
		throw new RuntimeException(e);
	}
}
 

private static TypeInformation<?> convertArray(String location, JsonNode node, JsonNode root) {
	// validate items
	if (!node.has(ITEMS)) {
		throw new IllegalArgumentException(
			"Arrays must specify an '" + ITEMS + "' property in node: " + location);
	}
	final JsonNode items = node.get(ITEMS);

	// list (translated to object array)
	if (items.isObject()) {
		final TypeInformation<?> elementType = convertType(
			location + '/' + ITEMS,
			items,
			root);
		// result type might either be ObjectArrayTypeInfo or BasicArrayTypeInfo for Strings
		return Types.OBJECT_ARRAY(elementType);
	}
	// tuple (translated to row)
	else if (items.isArray()) {
		final TypeInformation<?>[] types = convertTypes(location + '/' + ITEMS, items, root);

		// validate that array does not contain additional items
		if (node.has(ADDITIONAL_ITEMS) && node.get(ADDITIONAL_ITEMS).isBoolean() &&
				node.get(ADDITIONAL_ITEMS).asBoolean()) {
			throw new IllegalArgumentException(
				"An array tuple must not allow additional items in node: " + location);
		}

		return Types.ROW(types);
	}
	throw new IllegalArgumentException(
		"Invalid type for '" + ITEMS + "' property in node: " + location);
}
 

private DeserializationRuntimeConverter createArrayConverter(ArrayType arrayType) {
	final DeserializationRuntimeConverter elementConverter = createNullableConverter(arrayType.getElementType());
	final Class<?> elementClass = LogicalTypeUtils.toInternalConversionClass(arrayType.getElementType());
	return jsonNode -> {
		final ArrayNode node = (ArrayNode) jsonNode;
		final Object[] array = (Object[]) Array.newInstance(elementClass, node.size());
		for (int i = 0; i < node.size(); i++) {
			final JsonNode innerNode = node.get(i);
			array[i] = elementConverter.convert(innerNode);
		}
		return new GenericArrayData(array);
	};
}
 

private double convertToDouble(JsonNode jsonNode) {
	if (jsonNode.isDouble()) {
		// avoid redundant toString and parseDouble, for better performance
		return jsonNode.asDouble();
	} else {
		return Double.parseDouble(jsonNode.asText().trim());
	}
}
 

@Override
public Row deserialize(byte[] message) throws IOException {
	try {
		final JsonNode root = objectMapper.readTree(message);
		return (Row) runtimeConverter.convert(objectMapper, root);
	} catch (Throwable t) {
		throw new IOException("Failed to deserialize JSON object.", t);
	}
}
 

static Optional<AwsCredentials> optionalAwsCredentials(JsonNode specNode) {
  final JsonNode awsCredentialsSpecNode = specNode.at(AWS_CREDENTIALS_POINTER);
  if (awsCredentialsSpecNode.isMissingNode()) {
    return Optional.empty();
  }

  final String type = Selectors.textAt(awsCredentialsSpecNode, Credentials.TYPE_POINTER);
  switch (type) {
    case Credentials.DEFAULT_TYPE:
      return Optional.of(AwsCredentials.fromDefaultProviderChain());
    case Credentials.BASIC_TYPE:
      return Optional.of(
          AwsCredentials.basic(
              Selectors.textAt(awsCredentialsSpecNode, Credentials.ACCESS_KEY_ID_POINTER),
              Selectors.textAt(awsCredentialsSpecNode, Credentials.SECRET_ACCESS_KEY_POINTER)));
    case Credentials.PROFILE_TYPE:
      final Optional<String> path =
          Selectors.optionalTextAt(awsCredentialsSpecNode, Credentials.PROFILE_PATH_POINTER);
      if (path.isPresent()) {
        return Optional.of(
            AwsCredentials.profile(
                Selectors.textAt(awsCredentialsSpecNode, Credentials.PROFILE_NAME_POINTER),
                path.get()));
      } else {
        return Optional.of(
            AwsCredentials.profile(
                Selectors.textAt(awsCredentialsSpecNode, Credentials.PROFILE_NAME_POINTER)));
      }
    default:
      final List<String> validValues =
          Arrays.asList(
              Credentials.DEFAULT_TYPE, Credentials.BASIC_TYPE, Credentials.PROFILE_TYPE);
      throw new IllegalArgumentException(
          "Invalid AWS credential type: "
              + type
              + "; valid values are ["
              + String.join(", ", validValues)
              + "]");
  }
}
 

private static JsonNode fromPath(String path) {
  URL moduleUrl = ProtobufKafkaSourceProvider.class.getClassLoader().getResource(path);
  ObjectMapper mapper = new ObjectMapper(new YAMLFactory());
  try {
    return mapper.readTree(moduleUrl);
  } catch (IOException e) {
    throw new RuntimeException(e);
  }
}