org.apache.flink.streaming.connectors.kafka.internals.AbstractFetcher Java Examples

The following examples show how to use org.apache.flink.streaming.connectors.kafka.internals.AbstractFetcher. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: FlinkKafkaConsumerBaseTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Override
protected AbstractFetcher<T, ?> createFetcher(
		SourceContext<T> sourceContext,
		Map<KafkaTopicPartition, Long> thisSubtaskPartitionsWithStartOffsets,
		SerializedValue<WatermarkStrategy<T>> watermarkStrategy,
		StreamingRuntimeContext runtimeContext,
		OffsetCommitMode offsetCommitMode,
		MetricGroup consumerMetricGroup,
		boolean useMetrics) throws Exception {
	return new TestingFetcher<T, String>(
			sourceContext,
			thisSubtaskPartitionsWithStartOffsets,
			watermarkStrategy,
			runtimeContext.getProcessingTimeService(),
			0L,
			getClass().getClassLoader(),
			consumerMetricGroup,
			useMetrics);
}
Example #2
Source File: FlinkKafkaConsumerBaseTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@SuppressWarnings("unchecked")
DummyFlinkKafkaConsumer(
	SupplierWithException<AbstractFetcher<T, ?>, Exception> testFetcherSupplier,
	AbstractPartitionDiscoverer testPartitionDiscoverer,
	boolean isAutoCommitEnabled,
	long discoveryIntervalMillis,
	List<String> topics,
	KafkaDeserializationSchema<T> mock) {

	super(
		topics,
		null,
		mock,
		discoveryIntervalMillis,
		false);

	this.testFetcherSupplier = testFetcherSupplier;
	this.testPartitionDiscoverer = testPartitionDiscoverer;
	this.isAutoCommitEnabled = isAutoCommitEnabled;
}
Example #3
Source File: FlinkKafkaConsumerBaseTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void testClosePartitionDiscovererWhenKafkaFetcherFails() throws Exception {
	final FlinkException failureCause = new FlinkException("Run Kafka fetcher failure.");

	// in this scenario, the partition discoverer will be concurrently accessed;
	// use the WakeupBeforeCloseTestingPartitionDiscoverer to verify that we always call
	// wakeup() before closing the discoverer
	final WakeupBeforeCloseTestingPartitionDiscoverer testPartitionDiscoverer = new WakeupBeforeCloseTestingPartitionDiscoverer();

	final AbstractFetcher<String, ?> mock = (AbstractFetcher<String, ?>) mock(AbstractFetcher.class);
	doThrow(failureCause).when(mock).runFetchLoop();

	final DummyFlinkKafkaConsumer<String> consumer = new DummyFlinkKafkaConsumer<>(() -> mock, testPartitionDiscoverer, 100L);

	testFailingConsumerLifecycle(consumer, failureCause);
	assertTrue("partitionDiscoverer should be closed when consumer is closed", testPartitionDiscoverer.isClosed());
}
Example #4
Source File: FlinkKafkaConsumerBaseMigrationTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@SuppressWarnings("unchecked")
DummyFlinkKafkaConsumer(
		AbstractFetcher<T, ?> fetcher,
		List<String> topics,
		List<KafkaTopicPartition> partitions,
		long discoveryInterval) {

	super(
		topics,
		null,
		(KafkaDeserializationSchema< T >) mock(KafkaDeserializationSchema.class),
		discoveryInterval,
		false);

	this.fetcher = fetcher;
	this.partitions = partitions;
}
Example #5
Source File: FlinkKafkaConsumerBaseTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@SuppressWarnings("unchecked")
DummyFlinkKafkaConsumer(
	SupplierWithException<AbstractFetcher<T, ?>, Exception> testFetcherSupplier,
	AbstractPartitionDiscoverer testPartitionDiscoverer,
	boolean isAutoCommitEnabled,
	long discoveryIntervalMillis,
	List<String> topics) {

	super(
		topics,
		null,
		(KeyedDeserializationSchema< T >) mock(KeyedDeserializationSchema.class),
		discoveryIntervalMillis,
		false);

	this.testFetcherSupplier = testFetcherSupplier;
	this.testPartitionDiscoverer = testPartitionDiscoverer;
	this.isAutoCommitEnabled = isAutoCommitEnabled;
}
Example #6
Source File: FlinkKafkaConsumerBaseMigrationTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@SuppressWarnings("unchecked")
DummyFlinkKafkaConsumer(
		AbstractFetcher<T, ?> fetcher,
		List<String> topics,
		List<KafkaTopicPartition> partitions,
		long discoveryInterval) {

	super(
		topics,
		null,
		(KafkaDeserializationSchema< T >) mock(KafkaDeserializationSchema.class),
		discoveryInterval,
		false);

	this.fetcher = fetcher;
	this.partitions = partitions;
}
Example #7
Source File: FlinkKafkaConsumerBaseTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
@SuppressWarnings("unchecked")
DummyFlinkKafkaConsumer(
	SupplierWithException<AbstractFetcher<T, ?>, Exception> testFetcherSupplier,
	AbstractPartitionDiscoverer testPartitionDiscoverer,
	boolean isAutoCommitEnabled,
	long discoveryIntervalMillis,
	List<String> topics) {

	super(
		topics,
		null,
		(KeyedDeserializationSchema< T >) mock(KeyedDeserializationSchema.class),
		discoveryIntervalMillis,
		false);

	this.testFetcherSupplier = testFetcherSupplier;
	this.testPartitionDiscoverer = testPartitionDiscoverer;
	this.isAutoCommitEnabled = isAutoCommitEnabled;
}
Example #8
Source File: FlinkKafkaConsumerBaseTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
@Test
public void testClosePartitionDiscovererWhenKafkaFetcherFails() throws Exception {
	final FlinkException failureCause = new FlinkException("Run Kafka fetcher failure.");

	// in this scenario, the partition discoverer will be concurrently accessed;
	// use the WakeupBeforeCloseTestingPartitionDiscoverer to verify that we always call
	// wakeup() before closing the discoverer
	final WakeupBeforeCloseTestingPartitionDiscoverer testPartitionDiscoverer = new WakeupBeforeCloseTestingPartitionDiscoverer();

	final AbstractFetcher<String, ?> mock = (AbstractFetcher<String, ?>) mock(AbstractFetcher.class);
	doThrow(failureCause).when(mock).runFetchLoop();

	final DummyFlinkKafkaConsumer<String> consumer = new DummyFlinkKafkaConsumer<>(() -> mock, testPartitionDiscoverer, 100L);

	testFailingConsumerLifecycle(consumer, failureCause);
	assertTrue("partitionDiscoverer should be closed when consumer is closed", testPartitionDiscoverer.isClosed());
}
Example #9
Source File: FlinkKafkaConsumerBaseTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void testClosePartitionDiscovererWhenKafkaFetcherFails() throws Exception {
	final FlinkException failureCause = new FlinkException("Run Kafka fetcher failure.");

	// in this scenario, the partition discoverer will be concurrently accessed;
	// use the WakeupBeforeCloseTestingPartitionDiscoverer to verify that we always call
	// wakeup() before closing the discoverer
	final WakeupBeforeCloseTestingPartitionDiscoverer testPartitionDiscoverer = new WakeupBeforeCloseTestingPartitionDiscoverer();

	final AbstractFetcher<String, ?> mock = (AbstractFetcher<String, ?>) mock(AbstractFetcher.class);
	doThrow(failureCause).when(mock).runFetchLoop();

	final DummyFlinkKafkaConsumer<String> consumer = new DummyFlinkKafkaConsumer<>(() -> mock, testPartitionDiscoverer, 100L);

	testFailingConsumerLifecycle(consumer, failureCause);
	assertTrue("partitionDiscoverer should be closed when consumer is closed", testPartitionDiscoverer.isClosed());
}
Example #10
Source File: FlinkKafkaConsumerBaseMigrationTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
@SuppressWarnings("unchecked")
DummyFlinkKafkaConsumer(
		AbstractFetcher<T, ?> fetcher,
		List<String> topics,
		List<KafkaTopicPartition> partitions,
		long discoveryInterval) {

	super(
		topics,
		null,
		(KafkaDeserializationSchema< T >) mock(KafkaDeserializationSchema.class),
		discoveryInterval,
		false);

	this.fetcher = fetcher;
	this.partitions = partitions;
}
Example #11
Source File: FlinkKafkaConsumer010.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
protected AbstractFetcher<T, ?> createFetcher(
		SourceContext<T> sourceContext,
		Map<KafkaTopicPartition, Long> assignedPartitionsWithInitialOffsets,
		SerializedValue<WatermarkStrategy<T>> watermarkStrategy,
		StreamingRuntimeContext runtimeContext,
		OffsetCommitMode offsetCommitMode,
		MetricGroup consumerMetricGroup,
		boolean useMetrics) throws Exception {

	// make sure that auto commit is disabled when our offset commit mode is ON_CHECKPOINTS;
	// this overwrites whatever setting the user configured in the properties
	adjustAutoCommitConfig(properties, offsetCommitMode);

	// If a rateLimiter is set, then call rateLimiter.open() with the runtime context.
	if (rateLimiter != null) {
		rateLimiter.open(runtimeContext);
	}

	return new Kafka010Fetcher<>(
			sourceContext,
			assignedPartitionsWithInitialOffsets,
			watermarkStrategy,
			runtimeContext.getProcessingTimeService(),
			runtimeContext.getExecutionConfig().getAutoWatermarkInterval(),
			runtimeContext.getUserCodeClassLoader(),
			runtimeContext.getTaskNameWithSubtasks(),
			deserializer,
			properties,
			pollTimeout,
			runtimeContext.getMetricGroup(),
			consumerMetricGroup,
			useMetrics,
			rateLimiter);
}
Example #12
Source File: FlinkKafkaConsumerBaseTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@SuppressWarnings("unchecked")
DummyFlinkKafkaConsumer(KafkaDeserializationSchema<T> kafkaDeserializationSchema) {
	this(
		() -> mock(AbstractFetcher.class),
		mock(AbstractPartitionDiscoverer.class),
		false,
		PARTITION_DISCOVERY_DISABLED,
		Collections.singletonList("dummy-topic"),
		kafkaDeserializationSchema);
}
Example #13
Source File: FlinkKafkaConsumerBaseTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@SuppressWarnings("unchecked")
DummyFlinkKafkaConsumer(
		AbstractFetcher<T, ?> testFetcher,
		AbstractPartitionDiscoverer testPartitionDiscoverer,
		boolean isAutoCommitEnabled) {
	this(
		testFetcher,
		testPartitionDiscoverer,
		isAutoCommitEnabled,
		PARTITION_DISCOVERY_DISABLED);
}
Example #14
Source File: FlinkKafkaConsumer08.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
@Override
protected AbstractFetcher<T, ?> createFetcher(
		SourceContext<T> sourceContext,
		Map<KafkaTopicPartition, Long> assignedPartitionsWithInitialOffsets,
		SerializedValue<AssignerWithPeriodicWatermarks<T>> watermarksPeriodic,
		SerializedValue<AssignerWithPunctuatedWatermarks<T>> watermarksPunctuated,
		StreamingRuntimeContext runtimeContext,
		OffsetCommitMode offsetCommitMode,
		MetricGroup consumerMetricGroup,
		boolean useMetrics) throws Exception {

	long autoCommitInterval = (offsetCommitMode == OffsetCommitMode.KAFKA_PERIODIC)
			? PropertiesUtil.getLong(kafkaProperties, "auto.commit.interval.ms", 60000)
			: -1; // this disables the periodic offset committer thread in the fetcher

	return new Kafka08Fetcher<>(
			sourceContext,
			assignedPartitionsWithInitialOffsets,
			watermarksPeriodic,
			watermarksPunctuated,
			runtimeContext,
			deserializer,
			kafkaProperties,
			autoCommitInterval,
			consumerMetricGroup,
			useMetrics);
}
Example #15
Source File: FlinkKafkaConsumerBaseTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
protected AbstractFetcher<T, ?> createFetcher(
		SourceContext<T> sourceContext,
		Map<KafkaTopicPartition, Long> thisSubtaskPartitionsWithStartOffsets,
		SerializedValue<WatermarkStrategy<T>> watermarkStrategy,
		StreamingRuntimeContext runtimeContext,
		OffsetCommitMode offsetCommitMode,
		MetricGroup consumerMetricGroup,
		boolean useMetrics) throws Exception {
	return testFetcherSupplier.get();
}
Example #16
Source File: FlinkKafkaConsumer.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
@Override
protected AbstractFetcher<T, ?> createFetcher(
	SourceContext<T> sourceContext,
	Map<KafkaTopicPartition, Long> assignedPartitionsWithInitialOffsets,
	SerializedValue<AssignerWithPeriodicWatermarks<T>> watermarksPeriodic,
	SerializedValue<AssignerWithPunctuatedWatermarks<T>> watermarksPunctuated,
	StreamingRuntimeContext runtimeContext,
	OffsetCommitMode offsetCommitMode,
	MetricGroup consumerMetricGroup,
	boolean useMetrics) throws Exception {

	// make sure that auto commit is disabled when our offset commit mode is ON_CHECKPOINTS;
	// this overwrites whatever setting the user configured in the properties
	adjustAutoCommitConfig(properties, offsetCommitMode);

	return new KafkaFetcher<>(
		sourceContext,
		assignedPartitionsWithInitialOffsets,
		watermarksPeriodic,
		watermarksPunctuated,
		runtimeContext.getProcessingTimeService(),
		runtimeContext.getExecutionConfig().getAutoWatermarkInterval(),
		runtimeContext.getUserCodeClassLoader(),
		runtimeContext.getTaskNameWithSubtasks(),
		deserializer,
		properties,
		pollTimeout,
		runtimeContext.getMetricGroup(),
		consumerMetricGroup,
		useMetrics);
}
Example #17
Source File: FlinkKafkaConsumerBaseTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@SuppressWarnings("unchecked")
DummyFlinkKafkaConsumer(
		SupplierWithException<AbstractFetcher<T, ?>, Exception> testFetcherSupplier,
		AbstractPartitionDiscoverer testPartitionDiscoverer,
		boolean isAutoCommitEnabled,
		long discoveryIntervalMillis) {
	this(
		testFetcherSupplier,
		testPartitionDiscoverer,
		isAutoCommitEnabled,
		discoveryIntervalMillis,
		Collections.singletonList("dummy-topic"),
		(KeyedDeserializationSchema<T>) mock(KeyedDeserializationSchema.class)
	);
}
Example #18
Source File: FlinkKafkaConsumerBaseTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@SuppressWarnings("unchecked")
DummyFlinkKafkaConsumer(
		AbstractFetcher<T, ?> testFetcher,
		AbstractPartitionDiscoverer testPartitionDiscoverer,
		boolean isAutoCommitEnabled,
		long discoveryIntervalMillis) {
	this(
		() -> testFetcher,
		testPartitionDiscoverer,
		isAutoCommitEnabled,
		discoveryIntervalMillis);
}
Example #19
Source File: FlinkKafkaConsumerBaseTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@SuppressWarnings("unchecked")
DummyFlinkKafkaConsumer(List<String> topics, AbstractPartitionDiscoverer abstractPartitionDiscoverer) {
	this(
		() -> mock(AbstractFetcher.class),
		abstractPartitionDiscoverer,
		false,
		PARTITION_DISCOVERY_DISABLED,
		topics,
		(KeyedDeserializationSchema<T>) mock(KeyedDeserializationSchema.class));
}
Example #20
Source File: FlinkKafkaConsumerBaseMigrationTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
@Override
protected AbstractFetcher<T, ?> createFetcher(
		SourceContext<T> sourceContext,
		Map<KafkaTopicPartition, Long> thisSubtaskPartitionsWithStartOffsets,
		SerializedValue<AssignerWithPeriodicWatermarks<T>> watermarksPeriodic,
		SerializedValue<AssignerWithPunctuatedWatermarks<T>> watermarksPunctuated,
		StreamingRuntimeContext runtimeContext,
		OffsetCommitMode offsetCommitMode,
		MetricGroup consumerMetricGroup,
		boolean useMetrics) throws Exception {
	return fetcher;
}
Example #21
Source File: FlinkKafkaConsumer.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
protected AbstractFetcher<T, ?> createFetcher(
	SourceContext<T> sourceContext,
	Map<KafkaTopicPartition, Long> assignedPartitionsWithInitialOffsets,
	SerializedValue<WatermarkStrategy<T>> watermarkStrategy,
	StreamingRuntimeContext runtimeContext,
	OffsetCommitMode offsetCommitMode,
	MetricGroup consumerMetricGroup,
	boolean useMetrics) throws Exception {

	// make sure that auto commit is disabled when our offset commit mode is ON_CHECKPOINTS;
	// this overwrites whatever setting the user configured in the properties
	adjustAutoCommitConfig(properties, offsetCommitMode);

	return new KafkaFetcher<>(
		sourceContext,
		assignedPartitionsWithInitialOffsets,
		watermarkStrategy,
		runtimeContext.getProcessingTimeService(),
		runtimeContext.getExecutionConfig().getAutoWatermarkInterval(),
		runtimeContext.getUserCodeClassLoader(),
		runtimeContext.getTaskNameWithSubtasks(),
		deserializer,
		properties,
		pollTimeout,
		runtimeContext.getMetricGroup(),
		consumerMetricGroup,
		useMetrics);
}
Example #22
Source File: FlinkKafkaShuffleConsumer.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
protected AbstractFetcher<T, ?> createFetcher(
		SourceContext<T> sourceContext,
		Map<KafkaTopicPartition, Long> assignedPartitionsWithInitialOffsets,
		SerializedValue<WatermarkStrategy<T>> watermarkStrategy,
		StreamingRuntimeContext runtimeContext,
		OffsetCommitMode offsetCommitMode,
		MetricGroup consumerMetricGroup,
		boolean useMetrics) throws Exception {
	// make sure that auto commit is disabled when our offset commit mode is ON_CHECKPOINTS;
	// this overwrites whatever setting the user configured in the properties
	adjustAutoCommitConfig(properties, offsetCommitMode);

	return new KafkaShuffleFetcher<>(
		sourceContext,
		assignedPartitionsWithInitialOffsets,
		watermarkStrategy,
		runtimeContext.getProcessingTimeService(),
		runtimeContext.getExecutionConfig().getAutoWatermarkInterval(),
		runtimeContext.getUserCodeClassLoader(),
		runtimeContext.getTaskNameWithSubtasks(),
		deserializer,
		properties,
		pollTimeout,
		runtimeContext.getMetricGroup(),
		consumerMetricGroup,
		useMetrics,
		typeSerializer,
		producerParallelism);
}
Example #23
Source File: FlinkKafkaConsumerBaseTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
@SuppressWarnings("unchecked")
protected AbstractFetcher<T, ?> createFetcher(
		SourceContext<T> sourceContext,
		Map<KafkaTopicPartition, Long> thisSubtaskPartitionsWithStartOffsets,
		SerializedValue<AssignerWithPeriodicWatermarks<T>> watermarksPeriodic,
		SerializedValue<AssignerWithPunctuatedWatermarks<T>> watermarksPunctuated,
		StreamingRuntimeContext runtimeContext,
		OffsetCommitMode offsetCommitMode,
		MetricGroup consumerMetricGroup,
		boolean useMetrics) throws Exception {
	return testFetcherSupplier.get();
}
Example #24
Source File: FlinkKafkaConsumerBaseTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@SuppressWarnings("unchecked")
DummyFlinkKafkaConsumer(
		SupplierWithException<AbstractFetcher<T, ?>, Exception> testFetcherSupplier,
		AbstractPartitionDiscoverer testPartitionDiscoverer,
		boolean isAutoCommitEnabled,
		long discoveryIntervalMillis) {
	this(
		testFetcherSupplier,
		testPartitionDiscoverer,
		isAutoCommitEnabled,
		discoveryIntervalMillis,
		Collections.singletonList("dummy-topic")
		);
}
Example #25
Source File: FlinkKafkaConsumerBaseTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@SuppressWarnings("unchecked")
DummyFlinkKafkaConsumer(
		AbstractFetcher<T, ?> testFetcher,
		AbstractPartitionDiscoverer testPartitionDiscoverer,
		boolean isAutoCommitEnabled,
		long discoveryIntervalMillis) {
	this(
		() -> testFetcher,
		testPartitionDiscoverer,
		isAutoCommitEnabled,
		discoveryIntervalMillis);
}
Example #26
Source File: FlinkKafkaConsumerBaseTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@SuppressWarnings("unchecked")
DummyFlinkKafkaConsumer(
		AbstractFetcher<T, ?> testFetcher,
		AbstractPartitionDiscoverer testPartitionDiscoverer,
		boolean isAutoCommitEnabled) {
	this(
		testFetcher,
		testPartitionDiscoverer,
		isAutoCommitEnabled,
		PARTITION_DISCOVERY_DISABLED);
}
Example #27
Source File: FlinkKafkaConsumerBaseTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@SuppressWarnings("unchecked")
DummyFlinkKafkaConsumer(List<String> topics, AbstractPartitionDiscoverer abstractPartitionDiscoverer) {
	this(
		() -> mock(AbstractFetcher.class),
		abstractPartitionDiscoverer,
		false,
		PARTITION_DISCOVERY_DISABLED,
		topics);
}
Example #28
Source File: FlinkKafkaConsumer09.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
@Override
protected AbstractFetcher<T, ?> createFetcher(
		SourceContext<T> sourceContext,
		Map<KafkaTopicPartition, Long> assignedPartitionsWithInitialOffsets,
		SerializedValue<AssignerWithPeriodicWatermarks<T>> watermarksPeriodic,
		SerializedValue<AssignerWithPunctuatedWatermarks<T>> watermarksPunctuated,
		StreamingRuntimeContext runtimeContext,
		OffsetCommitMode offsetCommitMode,
		MetricGroup consumerMetricGroup,
		boolean useMetrics) throws Exception {

	// make sure that auto commit is disabled when our offset commit mode is ON_CHECKPOINTS;
	// this overwrites whatever setting the user configured in the properties
	adjustAutoCommitConfig(properties, offsetCommitMode);

	// If a rateLimiter is set, then call rateLimiter.open() with the runtime context.
	if (rateLimiter != null) {
		rateLimiter.open(runtimeContext);
	}

	return new Kafka09Fetcher<>(
			sourceContext,
			assignedPartitionsWithInitialOffsets,
			watermarksPeriodic,
			watermarksPunctuated,
			runtimeContext.getProcessingTimeService(),
			runtimeContext.getExecutionConfig().getAutoWatermarkInterval(),
			runtimeContext.getUserCodeClassLoader(),
			runtimeContext.getTaskNameWithSubtasks(),
			deserializer,
			properties,
			pollTimeout,
			runtimeContext.getMetricGroup(),
			consumerMetricGroup,
			useMetrics,
			rateLimiter);
}
Example #29
Source File: FlinkKafkaConsumerBaseMigrationTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
protected AbstractFetcher<T, ?> createFetcher(
		SourceContext<T> sourceContext,
		Map<KafkaTopicPartition, Long> thisSubtaskPartitionsWithStartOffsets,
		SerializedValue<AssignerWithPeriodicWatermarks<T>> watermarksPeriodic,
		SerializedValue<AssignerWithPunctuatedWatermarks<T>> watermarksPunctuated,
		StreamingRuntimeContext runtimeContext,
		OffsetCommitMode offsetCommitMode,
		MetricGroup consumerMetricGroup,
		boolean useMetrics) throws Exception {
	return fetcher;
}
Example #30
Source File: FlinkKafkaConsumer010.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
@Override
protected AbstractFetcher<T, ?> createFetcher(
		SourceContext<T> sourceContext,
		Map<KafkaTopicPartition, Long> assignedPartitionsWithInitialOffsets,
		SerializedValue<AssignerWithPeriodicWatermarks<T>> watermarksPeriodic,
		SerializedValue<AssignerWithPunctuatedWatermarks<T>> watermarksPunctuated,
		StreamingRuntimeContext runtimeContext,
		OffsetCommitMode offsetCommitMode,
		MetricGroup consumerMetricGroup,
		boolean useMetrics) throws Exception {

	// make sure that auto commit is disabled when our offset commit mode is ON_CHECKPOINTS;
	// this overwrites whatever setting the user configured in the properties
	adjustAutoCommitConfig(properties, offsetCommitMode);

	FlinkConnectorRateLimiter rateLimiter = super.getRateLimiter();
	// If a rateLimiter is set, then call rateLimiter.open() with the runtime context.
	if (rateLimiter != null) {
		rateLimiter.open(runtimeContext);
	}

	return new Kafka010Fetcher<>(
			sourceContext,
			assignedPartitionsWithInitialOffsets,
			watermarksPeriodic,
			watermarksPunctuated,
			runtimeContext.getProcessingTimeService(),
			runtimeContext.getExecutionConfig().getAutoWatermarkInterval(),
			runtimeContext.getUserCodeClassLoader(),
			runtimeContext.getTaskNameWithSubtasks(),
			deserializer,
			properties,
			pollTimeout,
			runtimeContext.getMetricGroup(),
			consumerMetricGroup,
			useMetrics,
			rateLimiter);
}