org.apache.flink.streaming.api.datastream.DataStreamSink Java Examples

public static void main(String[] args) throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	SiteToSiteClientConfig clientConfig = new SiteToSiteClient.Builder()
			.url("http://localhost:8080/nifi")
			.portName("Data from Flink")
			.buildConfig();

	DataStreamSink<String> dataStream = env.fromElements("one", "two", "three", "four", "five", "q")
			.addSink(new NiFiSink<>(clientConfig, new NiFiDataPacketBuilder<String>() {
				@Override
				public NiFiDataPacket createNiFiDataPacket(String s, RuntimeContext ctx) {
					return new StandardNiFiDataPacket(s.getBytes(ConfigConstants.DEFAULT_CHARSET),
						new HashMap<String, String>());
				}
			}));

	env.execute();
}
 

public static void main(String[] args) throws Exception {
    //FlumeSink send data
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

    FlumeEventBuilder<String> flumeEventBuilder = new FlumeEventBuilder<String>() {
        @Override
        public Event createFlumeEvent(String value, RuntimeContext ctx) {
            return EventBuilder.withBody(value, Charset.defaultCharset());
        }
    };

    FlumeSink<String> flumeSink = new FlumeSink<>(clientType, hostname, port, flumeEventBuilder, 1, 1, 1);

    // Note: parallelisms and FlumeSink batchSize
    // if every parallelism not enough batchSize, this parallelism not word FlumeThriftService output
    DataStreamSink<String> dataStream = env.fromElements("one", "two", "three", "four", "five")
            .addSink(flumeSink);

    env.execute();
}
 

public static void main(String[] args) throws Exception {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

    SiteToSiteClientConfig clientConfig = new SiteToSiteClient.Builder()
            .url("http://localhost:8080/nifi")
            .portName("Data from Flink")
            .buildConfig();

    DataStreamSink<String> dataStream = env.fromElements("one", "two", "three", "four", "five", "q")
            .addSink(new NiFiSink<>(clientConfig, new NiFiDataPacketBuilder<String>() {
                @Override
                public NiFiDataPacket createNiFiDataPacket(String s, RuntimeContext ctx) {
                    return new StandardNiFiDataPacket(s.getBytes(ConfigConstants.DEFAULT_CHARSET),
                            new HashMap<String, String>());
                }
            }));

    env.execute();
}
 

@Override
public DataStreamSink<?> consumeDataStream(DataStream<Row> dataStream) {
	if (!(dataStream.getType() instanceof RowTypeInfo)) {
		throw new TableException("No support for the type of the given DataStream: " + dataStream.getType());
	}

	CassandraRowSink sink = new CassandraRowSink(
		dataStream.getType().getArity(),
		cql,
		builder,
		CassandraSinkBaseConfig.newBuilder().build(),
		new NoOpCassandraFailureHandler());

	return dataStream
			.addSink(sink)
			.setParallelism(dataStream.getParallelism())
			.name(TableConnectorUtils.generateRuntimeName(this.getClass(), fieldNames));

}
 

@Override
public DataStreamSink<?> consumeDataStream(DataStream<Row> dataStream) {
	if (!(dataStream.getType() instanceof RowTypeInfo)) {
		throw new TableException("No support for the type of the given DataStream: " + dataStream.getType());
	}

	CassandraRowSink sink = new CassandraRowSink(
		dataStream.getType().getArity(),
		cql,
		builder,
		CassandraSinkBaseConfig.newBuilder().build(),
		new NoOpCassandraFailureHandler());

	return dataStream
			.addSink(sink)
			.setParallelism(dataStream.getParallelism())
			.name(TableConnectorUtils.generateRuntimeName(this.getClass(), fieldNames));

}
 

@Override
public DataStreamSink<?> consumeDataStream(DataStream<Tuple2<Boolean, Row>> dataStream) {
	Configuration hbaseClientConf = HBaseConfigurationUtil.getHBaseConfiguration();
	hbaseClientConf.set(HConstants.ZOOKEEPER_QUORUM, hbaseOptions.getZkQuorum());
	hbaseOptions.getZkNodeParent().ifPresent(v -> hbaseClientConf.set(HConstants.ZOOKEEPER_ZNODE_PARENT, v));
	HBaseSinkFunction sinkFunction = new HBaseSinkFunction(
		hbaseOptions.getTableName(),
		hbaseClientConf,
		new LegacyMutationConverter(hbaseTableSchema),
		writeOptions.getBufferFlushMaxSizeInBytes(),
		writeOptions.getBufferFlushMaxRows(),
		writeOptions.getBufferFlushIntervalMillis());
	return dataStream
		.addSink(sinkFunction)
		.setParallelism(dataStream.getParallelism())
		.name(TableConnectorUtils.generateRuntimeName(this.getClass(), tableSchema.getFieldNames()));
}
 

public static void main(String[] args) throws Exception {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

    SiteToSiteClientConfig clientConfig = new SiteToSiteClient.Builder()
            .url("http://localhost:8080/nifi")
            .portName("Data from Flink")
            .buildConfig();

    DataStreamSink<String> dataStream = env.fromElements("one", "two", "three", "four", "five", "q")
            .addSink(new NiFiSink<>(clientConfig, new NiFiDataPacketBuilder<String>() {
                @Override
                public NiFiDataPacket createNiFiDataPacket(String s, RuntimeContext ctx) {
                    return new StandardNiFiDataPacket(s.getBytes(ConfigConstants.DEFAULT_CHARSET),
                            new HashMap<String, String>());
                }
            }));

    env.execute();
}
 

@Override
public DataStreamSink<?> consumeDataStream(DataStream<Tuple2<Boolean, Row>> dataStream) {
	final ElasticsearchUpsertSinkFunction upsertFunction =
		new ElasticsearchUpsertSinkFunction(
			index,
			docType,
			keyDelimiter,
			keyNullLiteral,
			serializationSchema,
			contentType,
			requestFactory,
			keyFieldIndices);
	final SinkFunction<Tuple2<Boolean, Row>> sinkFunction = createSinkFunction(
		hosts,
		failureHandler,
		sinkOptions,
		upsertFunction);
	return dataStream.addSink(sinkFunction)
		.setParallelism(dataStream.getParallelism())
		.name(TableConnectorUtils.generateRuntimeName(this.getClass(), getFieldNames()));
}
 

public static void main(String[] args) throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	SiteToSiteClientConfig clientConfig = new SiteToSiteClient.Builder()
			.url("http://localhost:8080/nifi")
			.portName("Data from Flink")
			.buildConfig();

	DataStreamSink<String> dataStream = env.fromElements("one", "two", "three", "four", "five", "q")
			.addSink(new NiFiSink<>(clientConfig, new NiFiDataPacketBuilder<String>() {
				@Override
				public NiFiDataPacket createNiFiDataPacket(String s, RuntimeContext ctx) {
					return new StandardNiFiDataPacket(s.getBytes(ConfigConstants.DEFAULT_CHARSET),
						new HashMap<String, String>());
				}
			}));

	env.execute();
}
 

@Override
public DataStreamSink<?> consumeDataStream(DataStream<Tuple2<Boolean, Row>> dataStream) {
	Configuration hbaseClientConf = HBaseConfiguration.create();
	hbaseClientConf.set(HConstants.ZOOKEEPER_QUORUM, hbaseOptions.getZkQuorum());
	hbaseOptions.getZkNodeParent().ifPresent(v -> hbaseClientConf.set(HConstants.ZOOKEEPER_ZNODE_PARENT, v));
	HBaseUpsertSinkFunction sinkFunction = new HBaseUpsertSinkFunction(
		hbaseOptions.getTableName(),
		hbaseTableSchema,
		hbaseClientConf,
		writeOptions.getBufferFlushMaxSizeInBytes(),
		writeOptions.getBufferFlushMaxRows(),
		writeOptions.getBufferFlushIntervalMillis());
	return dataStream
		.addSink(sinkFunction)
		.setParallelism(dataStream.getParallelism())
		.name(TableConnectorUtils.generateRuntimeName(this.getClass(), tableSchema.getFieldNames()));
}
 

private FlinkKafkaProducer010Configuration(
		DataStreamSink<T> originalSink,
		DataStream<T> inputStream,
		FlinkKafkaProducer010<T> producer) {
	//noinspection unchecked
	super(inputStream, originalSink.getTransformation().getOperator());
	this.transformation = originalSink.getTransformation();
	this.producer = producer;
}
 

@Override
public <T> DataStreamSink<T> writeToKafkaWithTimestamps(DataStream<T> stream, String topic, KeyedSerializationSchema<T> serSchema, Properties props) {
	FlinkKafkaProducer011<T> prod = new FlinkKafkaProducer011<>(
		topic, serSchema, props, Optional.of(new FlinkFixedPartitioner<>()), producerSemantic, FlinkKafkaProducer011.DEFAULT_KAFKA_PRODUCERS_POOL_SIZE);

	prod.setWriteTimestampToKafka(true);

	return stream.addSink(prod);
}
 

@Override
public <T> DataStreamSink<T> produceIntoKafka(DataStream<T> stream, String topic, KeyedSerializationSchema<T> serSchema, Properties props, FlinkKafkaPartitioner<T> partitioner) {
	return stream.addSink(new FlinkKafkaProducer<T>(
		topic,
		serSchema,
		props,
		Optional.ofNullable(partitioner),
		producerSemantic,
		FlinkKafkaProducer.DEFAULT_KAFKA_PRODUCERS_POOL_SIZE));
}
 

@Deprecated
public abstract <T> DataStreamSink<T> produceIntoKafka(
	DataStream<T> stream,
	String topic,
	KeyedSerializationSchema<T> serSchema,
	Properties props,
	FlinkKafkaPartitioner<T> partitioner);
 

@Override
public <T> DataStreamSink<T> produceIntoKafka(DataStream<T> stream, String topic, KeyedSerializationSchema<T> serSchema, Properties props, FlinkKafkaPartitioner<T> partitioner) {
	return stream.addSink(new FlinkKafkaProducer<T>(
		topic,
		serSchema,
		props,
		Optional.ofNullable(partitioner),
		producerSemantic,
		FlinkKafkaProducer.DEFAULT_KAFKA_PRODUCERS_POOL_SIZE));
}
 

@Override
public DataStreamSink<?> consumeDataStream(DataStream<Tuple2<Boolean, Row>> stream) {
	// add sink
	return stream
		.addSink(new CollectSink<>(targetAddress, targetPort, serializer))
		.name("SQL Client Stream Collect Sink")
		.setParallelism(1);
}
 

@Override
public DataStreamSink<?> consumeDataStream(DataStream<Tuple2<Boolean, Row>> dataStream) {
    return dataStream.addSink(new SinkFunction<Tuple2<Boolean, Row>>() {
        @Override
        public void invoke(Tuple2<Boolean, Row> value, Context context) throws Exception {
            //自定义Sink
            // f0==true :插入新数据
            // f0==false:删除旧数据
            if (value.f0) {
                //可以写入MySQL、Kafka或者发HttpPost...根据具体情况开发
                System.out.println(value.f1);
            }
        }
    });
}
 

@Override
public DataStreamSink<?> consumeDataStream(DataStream<Tuple2<Boolean, Row>> dataStreamTuple) {
    KuduSink upsertKuduSink = new KuduSink(writerConfigBuilder.build(), tableInfo, new UpsertOperationMapper(getTableSchema().getFieldNames()));

    return dataStreamTuple
            .addSink(upsertKuduSink)
            .setParallelism(dataStreamTuple.getParallelism())
            .name(TableConnectorUtils.generateRuntimeName(this.getClass(), getTableSchema().getFieldNames()));
}
 

@Override
public <T> DataStreamSink<T> produceIntoKafka(
		DataStream<T> stream,
		String topic,
		SerializationSchema<T> serSchema,
		Properties props,
		FlinkKafkaPartitioner<T> partitioner) {
	return stream.addSink(new FlinkKafkaProducer<T>(
		topic,
		serSchema,
		props,
		partitioner,
		producerSemantic,
		FlinkKafkaProducer.DEFAULT_KAFKA_PRODUCERS_POOL_SIZE));
}
 

@Override
public <T> DataStreamSink<T> produceIntoKafka(DataStream<T> stream, String topic, KeyedSerializationSchema<T> serSchema, Properties props, FlinkKafkaPartitioner<T> partitioner) {
	return stream.addSink(new FlinkKafkaProducer<T>(
		topic,
		serSchema,
		props,
		Optional.ofNullable(partitioner),
		producerSemantic,
		FlinkKafkaProducer.DEFAULT_KAFKA_PRODUCERS_POOL_SIZE));
}
 

@Override
public <T> DataStreamSink<T> produceIntoKafka(
		DataStream<T> stream,
		String topic,
		SerializationSchema<T> serSchema,
		Properties props,
		FlinkKafkaPartitioner<T> partitioner) {
	return stream.addSink(new FlinkKafkaProducer011<>(
		topic,
		new KeyedSerializationSchemaWrapper<>(serSchema),
		props,
		Optional.ofNullable(partitioner),
		producerSemantic,
		FlinkKafkaProducer011.DEFAULT_KAFKA_PRODUCERS_POOL_SIZE));
}
 

@Override
public DataStreamSink<?> consumeDataStream(DataStream<Row> dataStream) {
    checkState(schema != null, "Table sink is not configured");
    FlinkPravegaWriter<Row> writer = writerFactory.apply(schema);
    return dataStream.addSink(writer)
            .setParallelism(dataStream.getParallelism())
            .name(TableConnectorUtils.generateRuntimeName(this.getClass(), getFieldNames()));
}
 

void consumeFrom(SingleOutputStreamOperator<?> mainOutput) {
  sideOutputs.forEach(
      (id, tag) -> {
        final DataStream<Object> sideOutputStream = mainOutput.getSideOutput(tag);

        DecoratedSink decoratedSink = sinks.get(id);
        @SuppressWarnings("unchecked")
        SinkFunction<Object> sink = (SinkFunction<Object>) decoratedSink.sink;

        DataStreamSink<Object> streamSink = sideOutputStream.addSink(sink);
        streamSink.name(decoratedSink.name);
        streamSink.uid(decoratedSink.uid);
      });
}
 

private FlinkKafkaProducer010Configuration(
		DataStreamSink<T> originalSink,
		DataStream<T> inputStream,
		FlinkKafkaProducer010<T> producer) {
	//noinspection unchecked
	super(inputStream, originalSink.getTransformation().getOperator());
	this.transformation = originalSink.getTransformation();
	this.producer = producer;
}
 

private FlinkKafkaProducer010Configuration(
		DataStreamSink<T> originalSink,
		DataStream<T> inputStream,
		FlinkKafkaProducer010<T> producer) {
	//noinspection unchecked
	super(inputStream, originalSink.getTransformation().getOperator());
	this.transformation = originalSink.getTransformation();
	this.producer = producer;
}
 

void consumeFrom(SingleOutputStreamOperator<?> mainOutput) {
  sideOutputs.forEach(
      (id, tag) -> {
        final DataStream<Object> sideOutputStream = mainOutput.getSideOutput(tag);

        DecoratedSink decoratedSink = sinks.get(id);
        @SuppressWarnings("unchecked")
        SinkFunction<Object> sink = (SinkFunction<Object>) decoratedSink.sink;

        DataStreamSink<Object> streamSink = sideOutputStream.addSink(sink);
        streamSink.name(decoratedSink.name);
        streamSink.uid(decoratedSink.uid);
      });
}
 

@Override
public <T> DataStreamSink<T> produceIntoKafka(
		DataStream<T> stream,
		String topic,
		SerializationSchema<T> serSchema,
		Properties props,
		FlinkKafkaPartitioner<T> partitioner) {
	FlinkKafkaProducer010<T> prod = new FlinkKafkaProducer010<>(topic, serSchema, props, partitioner);
	prod.setFlushOnCheckpoint(true);
	return stream.addSink(prod);
}
 

@Override
public DataStreamSink<?> consumeDataStream(DataStream<Row> dataStream) {
	final SinkFunction<Row> kafkaProducer = createKafkaProducer(
		topic,
		properties,
		serializationSchema,
		partitioner);
	return dataStream
		.addSink(kafkaProducer)
		.setParallelism(dataStream.getParallelism())
		.name(TableConnectorUtils.generateRuntimeName(this.getClass(), getFieldNames()));
}
 

@Override
public DataStreamSink<?> consumeDataStream(DataStream<Tuple2<Boolean, Row>> stream) {
	// add sink
	return stream
		.addSink(new CollectSink<>(targetAddress, targetPort, serializer))
		.name("SQL Client Stream Collect Sink")
		.setParallelism(1);
}
 

@Override
public DataStreamSink<?> consumeDataStream(DataStream<Tuple2<Boolean, Row>> dataStream) {
	return dataStream
			.addSink(new JDBCUpsertSinkFunction(newFormat()))
			.setParallelism(dataStream.getParallelism())
			.name(TableConnectorUtils.generateRuntimeName(this.getClass(), schema.getFieldNames()));
}