org.apache.flink.streaming.api.functions.sink.OutputFormatSinkFunction Java Examples

/**
 * Create a SimpleOperatorFactory from existed StreamOperator.
 */
@SuppressWarnings("unchecked")
public static <OUT> SimpleOperatorFactory<OUT> of(StreamOperator<OUT> operator) {
	if (operator == null) {
		return null;
	} else if (operator instanceof StreamSource &&
			((StreamSource) operator).getUserFunction() instanceof InputFormatSourceFunction) {
		return new SimpleInputFormatOperatorFactory<OUT>((StreamSource) operator);
	} else if (operator instanceof StreamSink &&
		((StreamSink) operator).getUserFunction() instanceof OutputFormatSinkFunction) {
		return new SimpleOutputFormatOperatorFactory<>((StreamSink) operator);
	} else if (operator instanceof AbstractUdfStreamOperator) {
		return new SimpleUdfStreamOperatorFactory<OUT>((AbstractUdfStreamOperator) operator);
	} else {
		return new SimpleOperatorFactory<>(operator);
	}
}
 

/**
 * Create a SimpleOperatorFactory from existed StreamOperator.
 */
@SuppressWarnings("unchecked")
public static <OUT> SimpleOperatorFactory<OUT> of(StreamOperator<OUT> operator) {
	if (operator == null) {
		return null;
	} else if (operator instanceof StreamSource &&
			((StreamSource) operator).getUserFunction() instanceof InputFormatSourceFunction) {
		return new SimpleInputFormatOperatorFactory<OUT>((StreamSource) operator);
	} else if (operator instanceof StreamSink &&
		((StreamSink) operator).getUserFunction() instanceof OutputFormatSinkFunction) {
		return new SimpleOutputFormatOperatorFactory<>((StreamSink) operator);
	} else if (operator instanceof AbstractUdfStreamOperator) {
		return new SimpleUdfStreamOperatorFactory<OUT>((AbstractUdfStreamOperator) operator);
	} else {
		return new SimpleOperatorFactory<>(operator);
	}
}
 

private OneInputStreamOperatorTestHarness<Row, Object> createSink(
		boolean override,
		boolean partition,
		boolean dynamicGrouped,
		LinkedHashMap<String, String> staticPartitions,
		AtomicReference<FileSystemOutputFormat<Row>> sinkRef) throws Exception {
	String[] columnNames = new String[]{"a", "b", "c"};
	String[] partitionColumns = partition ? new String[]{"c"} : new String[0];

	TableMetaStoreFactory msFactory = new FileSystemCommitterTest.TestMetaStoreFactory(
			new Path(outputFile.getPath()));
	FileSystemOutputFormat<Row> sink = new FileSystemOutputFormat.Builder<Row>()
			.setMetaStoreFactory(msFactory)
			.setTempPath(new Path(tmpFile.getPath()))
			.setOverwrite(override)
			.setPartitionColumns(partitionColumns)
			.setPartitionComputer(
					new RowPartitionComputer("default", columnNames, partitionColumns))
			.setFormatFactory(TextOutputFormat::new)
			.setDynamicGrouped(dynamicGrouped)
			.setStaticPartitions(staticPartitions)
			.build();

	sinkRef.set(sink);

	return new OneInputStreamOperatorTestHarness<>(
			new StreamSink<>(new OutputFormatSinkFunction<>(sink)),
			// test parallelism
			3, 3, 0);
}
 

public SimpleOutputFormatOperatorFactory(StreamSink<IN> operator) {
	super(operator);

	checkState(operator.getUserFunction() instanceof OutputFormatSinkFunction);
	this.operator = operator;
}
 

@Override
public OutputFormat<IN> getOutputFormat() {
	return ((OutputFormatSinkFunction<IN>) operator.getUserFunction()).getFormat();
}
 

@Override
public void emitDataStream(DataStream<Tuple2<Boolean, Row>> dataStream) {
    OutputFormatSinkFunction hbaseSink = creatHbaseSink();
    dataStream.addSink(hbaseSink);
}
 

private OutputFormatSinkFunction creatHbaseSink() {
    return new HbaseOutFormatSinkFunction(
        new HBaseOutputFormat(this.hbaseProperties, this.fieldNames, this.fieldTypes));
}
 

public SimpleOutputFormatOperatorFactory(StreamSink<IN> operator) {
	super(operator);

	checkState(operator.getUserFunction() instanceof OutputFormatSinkFunction);
	this.operator = operator;
}
 

@Override
public OutputFormat<IN> getOutputFormat() {
	return ((OutputFormatSinkFunction<IN>) operator.getUserFunction()).getFormat();
}
 

/**
 * Writes the dataStream into an output, described by an OutputFormat.
 *
 * <p>The output is not participating in Flink's checkpointing!
 *
 * <p>For writing to a file system periodically, the use of the "flink-connector-filesystem"
 * is recommended.
 *
 * @param format The output format
 * @return The closed DataStream
 */
@PublicEvolving
public DataStreamSink<T> writeUsingOutputFormat(OutputFormat<T> format) {
	return addSink(new OutputFormatSinkFunction<>(format));
}
 

/**
 * Writes the dataStream into an output, described by an OutputFormat.
 *
 * <p>The output is not participating in Flink's checkpointing!
 *
 * <p>For writing to a file system periodically, the use of the "flink-connector-filesystem"
 * is recommended.
 *
 * @param format The output format
 * @return The closed DataStream
 */
@PublicEvolving
public DataStreamSink<T> writeUsingOutputFormat(OutputFormat<T> format) {
	return addSink(new OutputFormatSinkFunction<>(format));
}
 

/**
 * Writes the dataStream into an output, described by an OutputFormat.
 *
 * <p>The output is not participating in Flink's checkpointing!
 *
 * <p>For writing to a file system periodically, the use of the "flink-connector-filesystem"
 * is recommended.
 *
 * @param format The output format
 * @return The closed DataStream
 *
 * @deprecated Please use the {@link org.apache.flink.streaming.api.functions.sink.filesystem.StreamingFileSink} explicitly using the
 * {@link #addSink(SinkFunction)} method.
 */
@Deprecated
@PublicEvolving
public DataStreamSink<T> writeUsingOutputFormat(OutputFormat<T> format) {
	return addSink(new OutputFormatSinkFunction<>(format));
}