org.apache.flink.streaming.api.functions.source.InputFormatSourceFunction Java Examples

/**
 * Transforms a {@code SourceTransformation}.
 */
private <T> Collection<Integer> transformSource(SourceTransformation<T> source) {
	String slotSharingGroup = determineSlotSharingGroup(source.getSlotSharingGroup(), Collections.emptyList());

	streamGraph.addSource(source.getId(),
			slotSharingGroup,
			source.getCoLocationGroupKey(),
			source.getOperator(),
			null,
			source.getOutputType(),
			"Source: " + source.getName());
	if (source.getOperator().getUserFunction() instanceof InputFormatSourceFunction) {
		InputFormatSourceFunction<T> fs = (InputFormatSourceFunction<T>) source.getOperator().getUserFunction();
		streamGraph.setInputFormat(source.getId(), fs.getFormat());
	}
	streamGraph.setParallelism(source.getId(), source.getParallelism());
	streamGraph.setMaxParallelism(source.getId(), source.getMaxParallelism());
	return Collections.singleton(source.getId());
}
 

/**
 * Create a SimpleOperatorFactory from existed StreamOperator.
 */
@SuppressWarnings("unchecked")
public static <OUT> SimpleOperatorFactory<OUT> of(StreamOperator<OUT> operator) {
	if (operator == null) {
		return null;
	} else if (operator instanceof StreamSource &&
			((StreamSource) operator).getUserFunction() instanceof InputFormatSourceFunction) {
		return new SimpleInputFormatOperatorFactory<OUT>((StreamSource) operator);
	} else if (operator instanceof StreamSink &&
		((StreamSink) operator).getUserFunction() instanceof OutputFormatSinkFunction) {
		return new SimpleOutputFormatOperatorFactory<>((StreamSink) operator);
	} else if (operator instanceof AbstractUdfStreamOperator) {
		return new SimpleUdfStreamOperatorFactory<OUT>((AbstractUdfStreamOperator) operator);
	} else {
		return new SimpleOperatorFactory<>(operator);
	}
}
 

/**
 * Create a SimpleOperatorFactory from existed StreamOperator.
 */
@SuppressWarnings("unchecked")
public static <OUT> SimpleOperatorFactory<OUT> of(StreamOperator<OUT> operator) {
	if (operator == null) {
		return null;
	} else if (operator instanceof StreamSource &&
			((StreamSource) operator).getUserFunction() instanceof InputFormatSourceFunction) {
		return new SimpleInputFormatOperatorFactory<OUT>((StreamSource) operator);
	} else if (operator instanceof StreamSink &&
		((StreamSink) operator).getUserFunction() instanceof OutputFormatSinkFunction) {
		return new SimpleOutputFormatOperatorFactory<>((StreamSink) operator);
	} else if (operator instanceof AbstractUdfStreamOperator) {
		return new SimpleUdfStreamOperatorFactory<OUT>((AbstractUdfStreamOperator) operator);
	} else {
		return new SimpleOperatorFactory<>(operator);
	}
}
 

private <OUT> DataStreamSource<OUT> createInput(InputFormat<OUT, ?> inputFormat,
												TypeInformation<OUT> typeInfo,
												String sourceName) {

	InputFormatSourceFunction<OUT> function = new InputFormatSourceFunction<>(inputFormat, typeInfo);
	return addSource(function, sourceName, typeInfo);
}
 

private <OUT> DataStreamSource<OUT> createInput(InputFormat<OUT, ?> inputFormat,
												TypeInformation<OUT> typeInfo,
												String sourceName) {

	InputFormatSourceFunction<OUT> function = new InputFormatSourceFunction<>(inputFormat, typeInfo);
	return addSource(function, sourceName, typeInfo);
}
 

@Override
public DataStream<RowData> getDataStream(StreamExecutionEnvironment execEnv) {
	@SuppressWarnings("unchecked")
	TypeInformation<RowData> typeInfo =
			(TypeInformation<RowData>) TypeInfoDataTypeConverter.fromDataTypeToTypeInfo(getProducedDataType());
	// Avoid using ContinuousFileMonitoringFunction
	InputFormatSourceFunction<RowData> func = new InputFormatSourceFunction<>(getInputFormat(), typeInfo);
	DataStreamSource<RowData> source = execEnv.addSource(func, explainSource(), typeInfo);
	return source.name(explainSource());
}
 

private <OUT> DataStreamSource<OUT> createInput(InputFormat<OUT, ?> inputFormat,
												TypeInformation<OUT> typeInfo,
												String sourceName) {

	InputFormatSourceFunction<OUT> function = new InputFormatSourceFunction<>(inputFormat, typeInfo);
	return addSource(function, sourceName, typeInfo);
}
 

public SimpleInputFormatOperatorFactory(StreamSource<OUT, InputFormatSourceFunction<OUT>> operator) {
	super(operator);
	this.operator = operator;
}
 

@Test
public void testInputOutputFormat() {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	DataStream<Long> source = env.addSource(
		new InputFormatSourceFunction<>(
			new TypeSerializerInputFormat<>(TypeInformation.of(Long.class)),
			TypeInformation.of(Long.class)),
		TypeInformation.of(Long.class)).name("source");

	source.writeUsingOutputFormat(new DiscardingOutputFormat<>()).name("sink1");
	source.writeUsingOutputFormat(new DiscardingOutputFormat<>()).name("sink2");

	StreamGraph streamGraph = env.getStreamGraph();
	JobGraph jobGraph = StreamingJobGraphGenerator.createJobGraph(streamGraph);
	assertEquals(1, jobGraph.getNumberOfVertices());

	JobVertex jobVertex = jobGraph.getVertices().iterator().next();
	assertTrue(jobVertex instanceof InputOutputFormatVertex);

	InputOutputFormatContainer formatContainer = new InputOutputFormatContainer(
		new TaskConfig(jobVertex.getConfiguration()), Thread.currentThread().getContextClassLoader());
	Map<OperatorID, UserCodeWrapper<? extends InputFormat<?, ?>>> inputFormats = formatContainer.getInputFormats();
	Map<OperatorID, UserCodeWrapper<? extends OutputFormat<?>>> outputFormats = formatContainer.getOutputFormats();
	assertEquals(1, inputFormats.size());
	assertEquals(2, outputFormats.size());

	Map<String, OperatorID> nameToOperatorIds = new HashMap<>();
	StreamConfig headConfig = new StreamConfig(jobVertex.getConfiguration());
	nameToOperatorIds.put(headConfig.getOperatorName(), headConfig.getOperatorID());

	Map<Integer, StreamConfig> chainedConfigs = headConfig
		.getTransitiveChainedTaskConfigs(Thread.currentThread().getContextClassLoader());
	for (StreamConfig config : chainedConfigs.values()) {
		nameToOperatorIds.put(config.getOperatorName(), config.getOperatorID());
	}

	InputFormat<?, ?> sourceFormat = inputFormats.get(nameToOperatorIds.get("Source: source")).getUserCodeObject();
	assertTrue(sourceFormat instanceof TypeSerializerInputFormat);

	OutputFormat<?> sinkFormat1 = outputFormats.get(nameToOperatorIds.get("Sink: sink1")).getUserCodeObject();
	assertTrue(sinkFormat1 instanceof DiscardingOutputFormat);

	OutputFormat<?> sinkFormat2 = outputFormats.get(nameToOperatorIds.get("Sink: sink2")).getUserCodeObject();
	assertTrue(sinkFormat2 instanceof DiscardingOutputFormat);
}
 

public SimpleInputFormatOperatorFactory(StreamSource<OUT, InputFormatSourceFunction<OUT>> operator) {
	super(operator);
	this.operator = operator;
}
 

@Test
public void testInputOutputFormat() {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	DataStream<Long> source = env.addSource(
		new InputFormatSourceFunction<>(
			new TypeSerializerInputFormat<>(TypeInformation.of(Long.class)),
			TypeInformation.of(Long.class)),
		TypeInformation.of(Long.class)).name("source");

	source.writeUsingOutputFormat(new DiscardingOutputFormat<>()).name("sink1");
	source.writeUsingOutputFormat(new DiscardingOutputFormat<>()).name("sink2");

	StreamGraph streamGraph = env.getStreamGraph();
	JobGraph jobGraph = StreamingJobGraphGenerator.createJobGraph(streamGraph);
	assertEquals(1, jobGraph.getNumberOfVertices());

	JobVertex jobVertex = jobGraph.getVertices().iterator().next();
	assertTrue(jobVertex instanceof InputOutputFormatVertex);

	InputOutputFormatContainer formatContainer = new InputOutputFormatContainer(
		new TaskConfig(jobVertex.getConfiguration()), Thread.currentThread().getContextClassLoader());
	Map<OperatorID, UserCodeWrapper<? extends InputFormat<?, ?>>> inputFormats = formatContainer.getInputFormats();
	Map<OperatorID, UserCodeWrapper<? extends OutputFormat<?>>> outputFormats = formatContainer.getOutputFormats();
	assertEquals(1, inputFormats.size());
	assertEquals(2, outputFormats.size());

	Map<String, OperatorID> nameToOperatorIds = new HashMap<>();
	StreamConfig headConfig = new StreamConfig(jobVertex.getConfiguration());
	nameToOperatorIds.put(headConfig.getOperatorName(), headConfig.getOperatorID());

	Map<Integer, StreamConfig> chainedConfigs = headConfig
		.getTransitiveChainedTaskConfigs(Thread.currentThread().getContextClassLoader());
	for (StreamConfig config : chainedConfigs.values()) {
		nameToOperatorIds.put(config.getOperatorName(), config.getOperatorID());
	}

	InputFormat<?, ?> sourceFormat = inputFormats.get(nameToOperatorIds.get("Source: source")).getUserCodeObject();
	assertTrue(sourceFormat instanceof TypeSerializerInputFormat);

	OutputFormat<?> sinkFormat1 = outputFormats.get(nameToOperatorIds.get("Sink: sink1")).getUserCodeObject();
	assertTrue(sinkFormat1 instanceof DiscardingOutputFormat);

	OutputFormat<?> sinkFormat2 = outputFormats.get(nameToOperatorIds.get("Sink: sink2")).getUserCodeObject();
	assertTrue(sinkFormat2 instanceof DiscardingOutputFormat);
}