org.apache.flink.streaming.api.functions.sink.SinkFunction Java Examples

@Override
public DataStreamSink<?> consumeDataStream(DataStream<Tuple2<Boolean, Row>> dataStream) {
	final ElasticsearchUpsertSinkFunction upsertFunction =
		new ElasticsearchUpsertSinkFunction(
			index,
			docType,
			keyDelimiter,
			keyNullLiteral,
			serializationSchema,
			contentType,
			requestFactory,
			keyFieldIndices);
	final SinkFunction<Tuple2<Boolean, Row>> sinkFunction = createSinkFunction(
		hosts,
		failureHandler,
		sinkOptions,
		upsertFunction);
	return dataStream.addSink(sinkFunction)
		.setParallelism(dataStream.getParallelism())
		.name(TableConnectorUtils.generateRuntimeName(this.getClass(), getFieldNames()));
}
 

@Override
public void emitDataStream(DataStream<Tuple2<Boolean, Row>> dataStream) {
	final ElasticsearchUpsertSinkFunction upsertFunction =
		new ElasticsearchUpsertSinkFunction(
			index,
			docType,
			keyDelimiter,
			keyNullLiteral,
			serializationSchema,
			contentType,
			requestFactory,
			keyFieldIndices);
	final SinkFunction<Tuple2<Boolean, Row>> sinkFunction = createSinkFunction(
		hosts,
		failureHandler,
		sinkOptions,
		upsertFunction);
	dataStream.addSink(sinkFunction)
		.name(TableConnectorUtils.generateRuntimeName(this.getClass(), getFieldNames()));
}
 

@Override
public <T> SinkFunction<T> forSpec(EgressSpec<T> egressSpec) {
  KafkaEgressSpec<T> spec = asSpec(egressSpec);

  Properties properties = new Properties();
  properties.putAll(spec.properties());
  properties.setProperty(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, spec.kafkaAddress());

  Semantic producerSemantic = semanticFromSpec(spec);
  if (producerSemantic == Semantic.EXACTLY_ONCE) {
    properties.setProperty(
        ProducerConfig.TRANSACTION_TIMEOUT_CONFIG,
        String.valueOf(spec.transactionTimeoutDuration().toMillis()));
  }

  return new FlinkKafkaProducer<>(
      randomKafkaTopic(),
      serializerFromSpec(spec),
      properties,
      producerSemantic,
      spec.kafkaProducerPoolSize());
}
 

public static void main(String[] args) throws Exception {

		final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
		env.setParallelism(4);

		DataStream<Tuple2<Long, Long>> stream = env.addSource(new DataSource());

		stream
			.keyBy(0)
			.timeWindow(Time.of(2500, MILLISECONDS), Time.of(500, MILLISECONDS))
			.reduce(new SummingReducer())

			// alternative: use a apply function which does not pre-aggregate
//			.keyBy(new FirstFieldKeyExtractor<Tuple2<Long, Long>, Long>())
//			.window(Time.of(2500, MILLISECONDS), Time.of(500, MILLISECONDS))
//			.apply(new SummingWindowFunction())

			.addSink(new SinkFunction<Tuple2<Long, Long>>() {
				@Override
				public void invoke(Tuple2<Long, Long> value) {
				}
			});

		env.execute();
	}
 

private DecoratedSink sinkFromSpec(EgressIdentifier<?> key, EgressSpec<?> spec) {
  SinkProvider provider = universe.sinks().get(spec.type());
  if (provider == null) {
    throw new IllegalStateException(
        "Unable to find a sink translation for egress of type "
            + spec.type()
            + ", which is bound for key "
            + key);
  }
  SinkFunction<?> sink = provider.forSpec(spec);
  if (sink == null) {
    throw new NullPointerException(
        "A sink provider for type " + spec.type() + ", has produced a NULL sink.");
  }
  return DecoratedSink.of(spec, sink);
}
 

@Test
public void jsonPathUdfTestReturn123()
        throws Exception
{
    String jsonKey = "$.key1.key2";
    //Table table = tableEnv.sqlQuery("select cast(json['store'] as double) from tp , LATERAL TABLE(json_parser(message, 'store', 'ip')) as T(json) ");
    Table table1 = tableEnv.sqlQuery("select get_json_object(message,'" + jsonKey + "') from " + table);
    tableEnv.toAppendStream(table1, Row.class)
            .addSink(new SinkFunction<Row>()
            {
                @Override
                public void invoke(Row value, Context context)
                        throws Exception
                {
                    result.put(jsonKey, (String) value.getField(0));
                }
            });
    tableEnv.execute("");
    Assert.assertEquals("123", result.get(jsonKey));
}
 

/**
 * Adds the given sink to this DataStream. Only streams with sinks added
 * will be executed once the {@link StreamExecutionEnvironment#execute()}
 * method is called.
 *
 * @param sinkFunction
 *            The object containing the sink's invoke function.
 * @return The closed DataStream.
 */
public DataStreamSink<T> addSink(SinkFunction<T> sinkFunction) {

	// read the output type of the input Transform to coax out errors about MissingTypeInfo
	transformation.getOutputType();

	// configure the type if needed
	if (sinkFunction instanceof InputTypeConfigurable) {
		((InputTypeConfigurable) sinkFunction).setInputType(getType(), getExecutionConfig());
	}

	StreamSink<T> sinkOperator = new StreamSink<>(clean(sinkFunction));

	DataStreamSink<T> sink = new DataStreamSink<>(this, sinkOperator);

	getExecutionEnvironment().addOperator(sink.getTransformation());
	return sink;
}
 

/**
 * Runs the following program.
 * <pre>
 *     [ (source)->(filter) ]-s->[ (map) ] -> [ (map) ] -> [ (groupBy/count)->(sink) ]
 * </pre>
 */
@Override
public void testProgram(StreamExecutionEnvironment env) {
	DataStream<String> stream = env.addSource(new StringGeneratingSourceFunction(NUM_STRINGS));

	stream
			// -------------- first vertex, chained to the source ----------------
			.filter(new StringRichFilterFunction())
			.shuffle()

			// -------------- seconds vertex - the stateful one that also fails ----------------
			.map(new StringPrefixCountRichMapFunction())
			.startNewChain()
			.map(new StatefulCounterFunction())

			// -------------- third vertex - counter and the sink ----------------
			.keyBy("prefix")
			.map(new OnceFailingPrefixCounter(NUM_STRINGS))
			.addSink(new SinkFunction<PrefixCount>() {

				@Override
				public void invoke(PrefixCount value) throws Exception {
					// Do nothing here
				}
			});
}
 

@Override
public void emitDataStream(DataStream<Tuple2<Boolean, Row>> dataStream) {
    Elasticsearch6SinkFunction upsertFunction = new Elasticsearch6SinkFunction(
            index,
            findIndex(indexField, schema.getFieldNames()),
            docType,
            keyDelimiter,
            keyNullLiteral,
            serializationSchema,
            contentType,
            requestFactory,
            keyFieldIndices);
    final SinkFunction<Tuple2<Boolean, Row>> sinkFunction = createSinkFunction(
            hosts,
            failureHandler,
            upsertFunction);
    dataStream.addSink(sinkFunction)
            .name(TableConnectorUtils.generateRuntimeName(this.getClass(), getFieldNames()));

}
 

private DecoratedSink sinkFromSpec(EgressIdentifier<?> key, EgressSpec<?> spec) {
  SinkProvider provider = universe.sinks().get(spec.type());
  if (provider == null) {
    throw new IllegalStateException(
        "Unable to find a sink translation for egress of type "
            + spec.type()
            + ", which is bound for key "
            + key);
  }
  SinkFunction<?> sink = provider.forSpec(spec);
  if (sink == null) {
    throw new NullPointerException(
        "A sink provider for type " + spec.type() + ", has produced a NULL sink.");
  }
  return DecoratedSink.of(spec, sink);
}
 

/**
 * Tests that the json generated by JSONGenerator shall meet with 2 requirements:
 * 1. sink nodes are at the back
 * 2. if both two nodes are sink nodes or neither of them is sink node, then sort by its id.
 */
@Test
public void testSinkIdComparison() {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	DataStream<Integer> source = env.fromElements(1, 2, 3);
	for (int i = 0; i < 32; i++) {
		if (i % 2 == 0) {
			source.addSink(new SinkFunction<Integer>() {
				@Override
				public void invoke(Integer value, Context ctx) throws Exception {}
			});
		} else {
			source.map(x -> x + 1);
		}
	}
	// IllegalArgumentException will be thrown without FLINK-9216
	env.getStreamGraph().getStreamingPlanAsJSON();
}
 

/**
 * Adds the given sink to this DataStream. Only streams with sinks added
 * will be executed once the {@link StreamExecutionEnvironment#execute()}
 * method is called.
 *
 * @param sinkFunction
 *            The object containing the sink's invoke function.
 * @return The closed DataStream.
 */
public DataStreamSink<T> addSink(SinkFunction<T> sinkFunction) {

	// read the output type of the input Transform to coax out errors about MissingTypeInfo
	transformation.getOutputType();

	// configure the type if needed
	if (sinkFunction instanceof InputTypeConfigurable) {
		((InputTypeConfigurable) sinkFunction).setInputType(getType(), getExecutionConfig());
	}

	StreamSink<T> sinkOperator = new StreamSink<>(clean(sinkFunction));

	DataStreamSink<T> sink = new DataStreamSink<>(this, sinkOperator);

	getExecutionEnvironment().addOperator(sink.getTransformation());
	return sink;
}
 

Bucket<IN, BucketID> onElement(final IN value, final SinkFunction.Context context) throws Exception {
	final long currentProcessingTime = context.currentProcessingTime();

	// setting the values in the bucketer context
	bucketerContext.update(
			context.timestamp(),
			context.currentWatermark(),
			currentProcessingTime);

	final BucketID bucketId = bucketAssigner.getBucketId(value, bucketerContext);
	final Bucket<IN, BucketID> bucket = getOrCreateBucketForBucketId(bucketId);
	bucket.write(value, currentProcessingTime);

	// we update the global max counter here because as buckets become inactive and
	// get removed from the list of active buckets, at the time when we want to create
	// another part file for the bucket, if we start from 0 we may overwrite previous parts.

	this.maxPartCounter = Math.max(maxPartCounter, bucket.getPartCounter());
	return bucket;
}
 

public static void main(String[] args) throws Exception {

		final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
		env.setParallelism(4);

		DataStream<Tuple2<Long, Long>> stream = env.addSource(new DataSource());

		stream
			.keyBy(0)
			.timeWindow(Time.of(2500, MILLISECONDS), Time.of(500, MILLISECONDS))
			.reduce(new SummingReducer())

			// alternative: use a apply function which does not pre-aggregate
//			.keyBy(new FirstFieldKeyExtractor<Tuple2<Long, Long>, Long>())
//			.window(Time.of(2500, MILLISECONDS), Time.of(500, MILLISECONDS))
//			.apply(new SummingWindowFunction())

			.addSink(new SinkFunction<Tuple2<Long, Long>>() {
				@Override
				public void invoke(Tuple2<Long, Long> value) {
				}
			});

		env.execute();
	}
 

/**
 * Adds the given sink to this DataStream. Only streams with sinks added
 * will be executed once the {@link StreamExecutionEnvironment#execute()}
 * method is called.
 *
 * @param sinkFunction
 *            The object containing the sink's invoke function.
 * @return The closed DataStream.
 */
public DataStreamSink<T> addSink(SinkFunction<T> sinkFunction) {

	// read the output type of the input Transform to coax out errors about MissingTypeInfo
	transformation.getOutputType();

	// configure the type if needed
	if (sinkFunction instanceof InputTypeConfigurable) {
		((InputTypeConfigurable) sinkFunction).setInputType(getType(), getExecutionConfig());
	}

	StreamSink<T> sinkOperator = new StreamSink<>(clean(sinkFunction));

	DataStreamSink<T> sink = new DataStreamSink<>(this, sinkOperator);

	getExecutionEnvironment().addOperator(sink.getTransformation());
	return sink;
}
 

private void createConsumerTopology(StreamExecutionEnvironment env, AMQSourceConfig<String> config) {
    AMQSource<String> source = new AMQSource<>(config);

    env.addSource(source)
        .addSink(new SinkFunction<String>() {
            final HashSet<Integer> set = new HashSet<>();
            @Override
            public void invoke(String value, Context context) throws Exception {
                int val = Integer.parseInt(value.split("-")[1]);
                set.add(val);

                if (set.size() == MESSAGES_NUM) {
                    throw new SuccessException();
                }
            }
        });
}
 

/**
 * Tests that the json generated by JSONGenerator shall meet with 2 requirements:
 * 1. sink nodes are at the back
 * 2. if both two nodes are sink nodes or neither of them is sink node, then sort by its id.
 */
@Test
public void testSinkIdComparison() {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	DataStream<Integer> source = env.fromElements(1, 2, 3);
	for (int i = 0; i < 32; i++) {
		if (i % 2 == 0) {
			source.addSink(new SinkFunction<Integer>() {
				@Override
				public void invoke(Integer value, Context ctx) throws Exception {}
			});
		} else {
			source.map(x -> x + 1);
		}
	}
	// IllegalArgumentException will be thrown without FLINK-9216
	env.getStreamGraph().getStreamingPlanAsJSON();
}
 

@Override
public <T> SinkFunction<T> forSpec(EgressSpec<T> egressSpec) {
  KafkaEgressSpec<T> spec = asSpec(egressSpec);

  Properties properties = new Properties();
  properties.putAll(spec.properties());
  properties.put("bootstrap.servers", spec.kafkaAddress());

  Semantic producerSemantic = semanticFromSpec(spec);
  if (producerSemantic == Semantic.EXACTLY_ONCE) {
    properties.put("transaction.timeout.ms", spec.transactionTimeoutDuration().toMillis());
  }

  return new FlinkKafkaProducer<>(
      randomKafkaTopic(),
      serializerFromSpec(spec),
      properties,
      producerSemantic,
      spec.kafkaProducerPoolSize());
}
 

public SinkOperator(
		SinkFunction<RowData> sinkFunction,
		int rowtimeFieldIndex,
		NotNullEnforcer notNullEnforcer,
		int[] notNullFieldIndices,
		String[] allFieldNames) {
	super(sinkFunction);
	this.rowtimeFieldIndex = rowtimeFieldIndex;
	this.notNullFieldIndices = notNullFieldIndices;
	this.notNullEnforcer = notNullEnforcer;
	this.notNullCheck = notNullFieldIndices.length > 0;
	this.allFieldNames = allFieldNames;
	chainingStrategy = ChainingStrategy.ALWAYS;
}
 

@Test
public void testPipeline() throws Exception {

	final String alertMask = "42";

	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	HeapMetrics alert1 = testStats(0.42);
	HeapMetrics regular1 = testStats(0.452);
	HeapMetrics regular2 = testStats(0.245);
	HeapMetrics alert2 = testStats(0.9423);

	DataStreamSource<HeapMetrics> testInput = env.fromElements(alert1, alert2, regular1, regular2);
	HeapMonitorPipeline.computeHeapAlerts(testInput, ParameterTool.fromArgs(new String[]{"--alertMask", alertMask}))
		.addSink(new SinkFunction<HeapAlert>() {
			@Override
			public void invoke(HeapAlert value, Context context) {
				testOutput.add(value);
			}
		})
		.setParallelism(1);

	env.execute();

	assertEquals(Sets.newHashSet(HeapAlert.maskRatioMatch(alertMask, alert1),
			HeapAlert.maskRatioMatch(alertMask, alert2)), testOutput);
}
 

@Override
protected SinkFunction<Row> createKafkaProducer(
		String topic,
		Properties properties,
		SerializationSchema<Row> serializationSchema,
		Optional<FlinkKafkaPartitioner<Row>> partitioner) {
	return new FlinkKafkaProducer011<>(
		topic,
		serializationSchema,
		properties,
		partitioner);
}
 

@Test
public void testUpateWatermark() throws Exception {
	final Configuration clientConfiguration = new Configuration();
	clientConfiguration.setInteger(RestOptions.RETRY_MAX_ATTEMPTS, 0);

	final StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment(
		flink.getRestAddress().get().getHost(),
		flink.getRestAddress().get().getPort(),
		clientConfiguration);

	env.addSource(new TestSourceFunction(new JobManagerWatermarkTracker("fakeId")))
		.addSink(new SinkFunction<Integer>() {});
	env.execute();
}
 

public static DecoratedSink of(EgressSpec<?> spec, SinkFunction<?> sink) {
  EgressIdentifier<?> identifier = spec.id();
  String name = String.format("%s-%s-egress", identifier.namespace(), identifier.name());
  String uid =
      String.format(
          "%s-%s-%s-%s-egress",
          spec.type().namespace(), spec.type().type(), identifier.namespace(), identifier.name());

  return new DecoratedSink(name, uid, sink);
}
 

@Override
public <T> SinkFunction<T> forSpec(EgressSpec<T> spec) {
  if (!(spec instanceof SinkFunctionSpec)) {
    throw new IllegalStateException("spec " + spec + " is not of type SourceFunctionSpec");
  }
  SinkFunctionSpec<T> casted = (SinkFunctionSpec<T>) spec;
  return casted.delegate();
}
 

private static <T> SinkFunction<T> consumingEgressSpec(EgressSpec<T> spec) {
  if (!(spec instanceof ConsumingEgressSpec)) {
    throw new IllegalArgumentException("Unable to provider a source for " + spec);
  }
  ConsumingEgressSpec<T> casted = (ConsumingEgressSpec<T>) spec;
  return new ConsumingSink<>(casted.consumer());
}
 

@Test
public void exampleUsage() {
  final KinesisEgressSpec<String> kinesisEgressSpec =
      KinesisEgressBuilder.forIdentifier(ID)
          .withAwsRegion("us-west-1")
          .withAwsCredentials(AwsCredentials.basic("access-key-id", "secret-access-key"))
          .withSerializer(TestSerializer.class)
          .build();

  final KinesisSinkProvider provider = new KinesisSinkProvider();
  final SinkFunction<String> sink = provider.forSpec(kinesisEgressSpec);

  assertThat(sink, instanceOf(FlinkKinesisProducer.class));
}
 

@Override
public DataStreamSink<T> addSink(SinkFunction<T> sinkFunction) {
	DataStreamSink<T> result = super.addSink(sinkFunction);
	result.getTransformation().setStateKeySelector(keySelector);
	result.getTransformation().setStateKeyType(keyType);
	return result;
}
 

public TimestampValidatingOperator() {
	super(new SinkFunction<Long>() {
		private static final long serialVersionUID = -6676565693361786524L;

		@Override
		public void invoke(Long value) throws Exception {
			throw new RuntimeException("Unexpected");
		}
	});
}
 

@Test
public void testUpateWatermark() throws Exception {
	final Configuration clientConfiguration = new Configuration();
	clientConfiguration.setInteger(RestOptions.RETRY_MAX_ATTEMPTS, 0);

	final StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment(
		flink.getRestAddress().get().getHost(),
		flink.getRestAddress().get().getPort(),
		clientConfiguration);

	env.addSource(new TestSourceFunction(new JobManagerWatermarkTracker("fakeId")))
		.addSink(new SinkFunction<Integer>() {});
	env.execute();
}
 

private static StreamExecutionEnvironment getSimpleJob() {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.generateSequence(1, 10000000)
			.addSink(new SinkFunction<Long>() {
				@Override
				public void invoke(Long value) {
				}
			});

	return env;
}