org.apache.flink.streaming.api.functions.source.SourceFunction.SourceContext Java Examples

@Override
public void handleSimple(ServerRequest serverRequest) {
    if (!(serverRequest.getData() instanceof TBase<?, ?>)) {
        throw new UnsupportedOperationException("data is not support type : " + serverRequest.getData());
    }

    final TBase<?, ?> tBase = (TBase<?, ?>) serverRequest.getData();
    final Map<String, String> metaInfo = new HashMap<>(serverRequest.getHeaderEntity().getEntityAll());
    final RawData rawData = new RawData(tBase, metaInfo);
    final SourceContext sourceContext = roundRobinSourceContext();

    if (sourceContext == null) {
        logger.warn("sourceContext is null.");
        return;
    }

    sourceContext.collect(rawData);

}
 

TestFetcher(
		SourceContext<T> sourceContext,
		Map<KafkaTopicPartition, Long> assignedPartitionsWithStartOffsets,
		SerializedValue<WatermarkStrategy<T>> watermarkStrategy,
		ProcessingTimeService processingTimeProvider,
		long autoWatermarkInterval) throws Exception {
	super(
			sourceContext,
			assignedPartitionsWithStartOffsets,
			watermarkStrategy,
			processingTimeProvider,
			autoWatermarkInterval,
			TestFetcher.class.getClassLoader(),
			new UnregisteredMetricsGroup(),
			false);
}
 

TestFetcher(
		SourceContext<T> sourceContext,
		Map<KafkaTopicPartition, Long> assignedPartitionsWithStartOffsets,
		SerializedValue<WatermarkStrategy<T>> watermarkStrategy,
		ProcessingTimeService processingTimeProvider,
		long autoWatermarkInterval,
		OneShotLatch fetchLoopWaitLatch,
		OneShotLatch stateIterationBlockLatch) throws Exception {

	super(
		sourceContext,
		assignedPartitionsWithStartOffsets,
		watermarkStrategy,
		processingTimeProvider,
		autoWatermarkInterval,
		TestFetcher.class.getClassLoader(),
		new UnregisteredMetricsGroup(),
		false);

	this.fetchLoopWaitLatch = fetchLoopWaitLatch;
	this.stateIterationBlockLatch = stateIterationBlockLatch;
}
 

TestFetcher(
		SourceContext<T> sourceContext,
		Map<KafkaTopicPartition, Long> assignedPartitionsWithStartOffsets,
		SerializedValue<WatermarkStrategy<T>> watermarkStrategy,
		ProcessingTimeService processingTimeProvider,
		long autoWatermarkInterval) throws Exception {

	this(
		sourceContext,
		assignedPartitionsWithStartOffsets,
		watermarkStrategy,
		processingTimeProvider,
		autoWatermarkInterval,
		null,
		null);
}
 

TestFetcher(
		SourceContext<T> sourceContext,
		Map<KafkaTopicPartition, Long> assignedPartitionsWithStartOffsets,
		SerializedValue<AssignerWithPeriodicWatermarks<T>> watermarksPeriodic,
		SerializedValue<AssignerWithPunctuatedWatermarks<T>> watermarksPunctuated,
		ProcessingTimeService processingTimeProvider,
		long autoWatermarkInterval) throws Exception {

	this(
		sourceContext,
		assignedPartitionsWithStartOffsets,
		watermarksPeriodic,
		watermarksPunctuated,
		processingTimeProvider,
		autoWatermarkInterval,
		null,
		null);
}
 

TestFetcher(
		SourceContext<T> sourceContext,
		Map<KafkaTopicPartition, Long> assignedPartitionsWithStartOffsets,
		SerializedValue<AssignerWithPeriodicWatermarks<T>> watermarksPeriodic,
		SerializedValue<AssignerWithPunctuatedWatermarks<T>> watermarksPunctuated,
		ProcessingTimeService processingTimeProvider,
		long autoWatermarkInterval,
		OneShotLatch fetchLoopWaitLatch,
		OneShotLatch stateIterationBlockLatch) throws Exception {

	super(
		sourceContext,
		assignedPartitionsWithStartOffsets,
		watermarksPeriodic,
		watermarksPunctuated,
		processingTimeProvider,
		autoWatermarkInterval,
		TestFetcher.class.getClassLoader(),
		new UnregisteredMetricsGroup(),
		false);

	this.fetchLoopWaitLatch = fetchLoopWaitLatch;
	this.stateIterationBlockLatch = stateIterationBlockLatch;
}
 

TestFetcher(
		SourceContext<T> sourceContext,
		Map<KafkaTopicPartition, Long> assignedPartitionsWithStartOffsets,
		SerializedValue<AssignerWithPeriodicWatermarks<T>> watermarksPeriodic,
		SerializedValue<AssignerWithPunctuatedWatermarks<T>> watermarksPunctuated,
		ProcessingTimeService processingTimeProvider,
		long autoWatermarkInterval,
		OneShotLatch fetchLoopWaitLatch,
		OneShotLatch stateIterationBlockLatch) throws Exception {

	super(
		sourceContext,
		assignedPartitionsWithStartOffsets,
		watermarksPeriodic,
		watermarksPunctuated,
		processingTimeProvider,
		autoWatermarkInterval,
		TestFetcher.class.getClassLoader(),
		new UnregisteredMetricsGroup(),
		false);

	this.fetchLoopWaitLatch = fetchLoopWaitLatch;
	this.stateIterationBlockLatch = stateIterationBlockLatch;
}
 

TestFetcher(
		SourceContext<T> sourceContext,
		Map<KafkaTopicPartition, Long> assignedPartitionsWithStartOffsets,
		SerializedValue<AssignerWithPeriodicWatermarks<T>> watermarksPeriodic,
		SerializedValue<AssignerWithPunctuatedWatermarks<T>> watermarksPunctuated,
		ProcessingTimeService processingTimeProvider,
		long autoWatermarkInterval) throws Exception {

	this(
		sourceContext,
		assignedPartitionsWithStartOffsets,
		watermarksPeriodic,
		watermarksPunctuated,
		processingTimeProvider,
		autoWatermarkInterval,
		null,
		null);
}
 

public Kafka08Fetcher(
		SourceContext<T> sourceContext,
		Map<KafkaTopicPartition, Long> seedPartitionsWithInitialOffsets,
		SerializedValue<AssignerWithPeriodicWatermarks<T>> watermarksPeriodic,
		SerializedValue<AssignerWithPunctuatedWatermarks<T>> watermarksPunctuated,
		StreamingRuntimeContext runtimeContext,
		KafkaDeserializationSchema<T> deserializer,
		Properties kafkaProperties,
		long autoCommitInterval,
		MetricGroup consumerMetricGroup,
		boolean useMetrics) throws Exception {
	super(
			sourceContext,
			seedPartitionsWithInitialOffsets,
			watermarksPeriodic,
			watermarksPunctuated,
			runtimeContext.getProcessingTimeService(),
			runtimeContext.getExecutionConfig().getAutoWatermarkInterval(),
			runtimeContext.getUserCodeClassLoader(),
			consumerMetricGroup,
			useMetrics);

	this.deserializer = checkNotNull(deserializer);
	this.kafkaConfig = checkNotNull(kafkaProperties);
	this.runtimeContext = runtimeContext;
	this.invalidOffsetBehavior = getInvalidOffsetBehavior(kafkaProperties);
	this.autoCommitInterval = autoCommitInterval;
}
 

@Test
public void testConcurrentPartitionsDiscoveryAndLoopFetching() throws Exception {
	// test data
	final KafkaTopicPartition testPartition = new KafkaTopicPartition("test", 42);

	// ----- create the test fetcher -----

	SourceContext<String> sourceContext = new TestSourceContext<>();
	Map<KafkaTopicPartition, Long> partitionsWithInitialOffsets =
		Collections.singletonMap(testPartition, KafkaTopicPartitionStateSentinel.GROUP_OFFSET);

	final OneShotLatch fetchLoopWaitLatch = new OneShotLatch();
	final OneShotLatch stateIterationBlockLatch = new OneShotLatch();

	final TestFetcher<String> fetcher = new TestFetcher<>(
		sourceContext,
		partitionsWithInitialOffsets,
		null, /* watermark strategy */
		new TestProcessingTimeService(),
		10,
		fetchLoopWaitLatch,
		stateIterationBlockLatch);

	// ----- run the fetcher -----

	final CheckedThread checkedThread = new CheckedThread() {
		@Override
		public void go() throws Exception {
			fetcher.runFetchLoop();
		}
	};
	checkedThread.start();

	// wait until state iteration begins before adding discovered partitions
	fetchLoopWaitLatch.await();
	fetcher.addDiscoveredPartitions(Collections.singletonList(testPartition));

	stateIterationBlockLatch.trigger();
	checkedThread.sync();
}
 

public Kafka010Fetcher(
		SourceContext<T> sourceContext,
		Map<KafkaTopicPartition, Long> assignedPartitionsWithInitialOffsets,
		SerializedValue<WatermarkStrategy<T>> watermarkStrategy,
		ProcessingTimeService processingTimeProvider,
		long autoWatermarkInterval,
		ClassLoader userCodeClassLoader,
		String taskNameWithSubtasks,
		KafkaDeserializationSchema<T> deserializer,
		Properties kafkaProperties,
		long pollTimeout,
		MetricGroup subtaskMetricGroup,
		MetricGroup consumerMetricGroup,
		boolean useMetrics,
		FlinkConnectorRateLimiter rateLimiter) throws Exception {
	super(
			sourceContext,
			assignedPartitionsWithInitialOffsets,
			watermarkStrategy,
			processingTimeProvider,
			autoWatermarkInterval,
			userCodeClassLoader,
			consumerMetricGroup,
			useMetrics);

	this.deserializer = deserializer;
	this.handover = new Handover();

	this.consumerThread = new KafkaConsumerThread(
			LOG,
			handover,
			kafkaProperties,
			unassignedPartitionsQueue,
			"Kafka 0.10 Fetcher for " + taskNameWithSubtasks,
			pollTimeout,
			useMetrics,
			consumerMetricGroup,
			subtaskMetricGroup,
			rateLimiter);
}
 

public ReceiverActor(SourceContext<Object> ctx,
          String urlOfPublisher,
          boolean autoAck) {
  this.ctx = ctx;
  this.urlOfPublisher = urlOfPublisher;
  this.autoAck = autoAck;
}
 

public ReceiverActor(SourceContext<Object> ctx,
                     String urlOfPublisher,
                     boolean autoAck) {
    this.ctx = ctx;
    this.urlOfPublisher = urlOfPublisher;
    this.autoAck = autoAck;
}
 

public final void withCheckpointLock(SourceContext<String> ctx) {
	if (checkpointLockTriggered) {
		return;
	}

	checkpointLockTriggered = true;
	onceWithCheckpointLock(ctx);
}
 

PeriodicWatermarkEmitter(
		List<KafkaTopicPartitionState<KPH>> allPartitions,
		SourceContext<?> emitter,
		ProcessingTimeService timerService,
		long autoWatermarkInterval) {
	this.allPartitions = checkNotNull(allPartitions);
	this.emitter = checkNotNull(emitter);
	this.timerService = checkNotNull(timerService);
	this.interval = autoWatermarkInterval;
	this.lastWatermarkTimestamp = Long.MIN_VALUE;
}
 

public Kafka010Fetcher(
		SourceContext<T> sourceContext,
		Map<KafkaTopicPartition, Long> assignedPartitionsWithInitialOffsets,
		SerializedValue<AssignerWithPeriodicWatermarks<T>> watermarksPeriodic,
		SerializedValue<AssignerWithPunctuatedWatermarks<T>> watermarksPunctuated,
		ProcessingTimeService processingTimeProvider,
		long autoWatermarkInterval,
		ClassLoader userCodeClassLoader,
		String taskNameWithSubtasks,
		KafkaDeserializationSchema<T> deserializer,
		Properties kafkaProperties,
		long pollTimeout,
		MetricGroup subtaskMetricGroup,
		MetricGroup consumerMetricGroup,
		boolean useMetrics,
		FlinkConnectorRateLimiter rateLimiter) throws Exception {
	super(
			sourceContext,
			assignedPartitionsWithInitialOffsets,
			watermarksPeriodic,
			watermarksPunctuated,
			processingTimeProvider,
			autoWatermarkInterval,
			userCodeClassLoader,
			taskNameWithSubtasks,
			deserializer,
			kafkaProperties,
			pollTimeout,
			subtaskMetricGroup,
			consumerMetricGroup,
			useMetrics, rateLimiter);
}
 

public Kafka08Fetcher(
		SourceContext<T> sourceContext,
		Map<KafkaTopicPartition, Long> seedPartitionsWithInitialOffsets,
		SerializedValue<AssignerWithPeriodicWatermarks<T>> watermarksPeriodic,
		SerializedValue<AssignerWithPunctuatedWatermarks<T>> watermarksPunctuated,
		StreamingRuntimeContext runtimeContext,
		KafkaDeserializationSchema<T> deserializer,
		Properties kafkaProperties,
		long autoCommitInterval,
		MetricGroup consumerMetricGroup,
		boolean useMetrics) throws Exception {
	super(
			sourceContext,
			seedPartitionsWithInitialOffsets,
			watermarksPeriodic,
			watermarksPunctuated,
			runtimeContext.getProcessingTimeService(),
			runtimeContext.getExecutionConfig().getAutoWatermarkInterval(),
			runtimeContext.getUserCodeClassLoader(),
			consumerMetricGroup,
			useMetrics);

	this.deserializer = checkNotNull(deserializer);
	this.kafkaConfig = checkNotNull(kafkaProperties);
	this.runtimeContext = runtimeContext;
	this.invalidOffsetBehavior = getInvalidOffsetBehavior(kafkaProperties);
	this.autoCommitInterval = autoCommitInterval;
}
 

public Kafka010Fetcher(
		SourceContext<T> sourceContext,
		Map<KafkaTopicPartition, Long> assignedPartitionsWithInitialOffsets,
		SerializedValue<AssignerWithPeriodicWatermarks<T>> watermarksPeriodic,
		SerializedValue<AssignerWithPunctuatedWatermarks<T>> watermarksPunctuated,
		ProcessingTimeService processingTimeProvider,
		long autoWatermarkInterval,
		ClassLoader userCodeClassLoader,
		String taskNameWithSubtasks,
		KafkaDeserializationSchema<T> deserializer,
		Properties kafkaProperties,
		long pollTimeout,
		MetricGroup subtaskMetricGroup,
		MetricGroup consumerMetricGroup,
		boolean useMetrics,
		FlinkConnectorRateLimiter rateLimiter) throws Exception {
	super(
			sourceContext,
			assignedPartitionsWithInitialOffsets,
			watermarksPeriodic,
			watermarksPunctuated,
			processingTimeProvider,
			autoWatermarkInterval,
			userCodeClassLoader,
			taskNameWithSubtasks,
			deserializer,
			kafkaProperties,
			pollTimeout,
			subtaskMetricGroup,
			consumerMetricGroup,
			useMetrics, rateLimiter);
}
 

PeriodicWatermarkEmitter(
		List<KafkaTopicPartitionState<KPH>> allPartitions,
		SourceContext<?> emitter,
		ProcessingTimeService timerService,
		long autoWatermarkInterval) {
	this.allPartitions = checkNotNull(allPartitions);
	this.emitter = checkNotNull(emitter);
	this.timerService = checkNotNull(timerService);
	this.interval = autoWatermarkInterval;
	this.lastWatermarkTimestamp = Long.MIN_VALUE;
}
 

public ReceiverActor(SourceContext<Object> ctx,
                     String urlOfPublisher,
                     boolean autoAck) {
    this.ctx = ctx;
    this.urlOfPublisher = urlOfPublisher;
    this.autoAck = autoAck;
}
 

PeriodicWatermarkEmitter(
        List<PulsarTopicState> allPartitions,
        SourceContext<?> emitter,
        ProcessingTimeService timerService,
        long autoWatermarkInterval) {
    this.allPartitions = checkNotNull(allPartitions);
    this.emitter = checkNotNull(emitter);
    this.timerService = checkNotNull(timerService);
    this.interval = autoWatermarkInterval;
    this.lastWatermarkTimestamp = Long.MIN_VALUE;
}
 

public PulsarFetcher(
        SourceContext<T> sourceContext,
        Map<String, MessageId> seedTopicsWithInitialOffsets,
        SerializedValue<AssignerWithPeriodicWatermarks<T>> watermarksPeriodic,
        SerializedValue<AssignerWithPunctuatedWatermarks<T>> watermarksPunctuated,
        ProcessingTimeService processingTimeProvider,
        long autoWatermarkInterval,
        ClassLoader userCodeClassLoader,
        StreamingRuntimeContext runtimeContext,
        ClientConfigurationData clientConf,
        Map<String, Object> readerConf,
        int pollTimeoutMs,
        DeserializationSchema<T> deserializer,
        PulsarMetadataReader metadataReader) throws Exception {
    this(
            sourceContext,
            seedTopicsWithInitialOffsets,
            watermarksPeriodic,
            watermarksPunctuated,
            processingTimeProvider,
            autoWatermarkInterval,
            userCodeClassLoader,
            runtimeContext,
            clientConf,
            readerConf,
            pollTimeoutMs,
            3, // commit retries before fail
            deserializer,
            metadataReader);
}
 

public void setStatHandlerTcpDispatchHandler(SourceContext<RawData> sourceContext) {
    agentStatHandler.addSourceContext(sourceContext);
    tcpDispatchHandler.setSimpletHandler(agentStatHandler);
}
 

protected AbstractFetcher(
		SourceContext<T> sourceContext,
		Map<KafkaTopicPartition, Long> seedPartitionsWithInitialOffsets,
		SerializedValue<AssignerWithPeriodicWatermarks<T>> watermarksPeriodic,
		SerializedValue<AssignerWithPunctuatedWatermarks<T>> watermarksPunctuated,
		ProcessingTimeService processingTimeProvider,
		long autoWatermarkInterval,
		ClassLoader userCodeClassLoader,
		MetricGroup consumerMetricGroup,
		boolean useMetrics) throws Exception {
	this.sourceContext = checkNotNull(sourceContext);
	this.checkpointLock = sourceContext.getCheckpointLock();
	this.userCodeClassLoader = checkNotNull(userCodeClassLoader);

	this.useMetrics = useMetrics;
	this.consumerMetricGroup = checkNotNull(consumerMetricGroup);
	this.legacyCurrentOffsetsMetricGroup = consumerMetricGroup.addGroup(LEGACY_CURRENT_OFFSETS_METRICS_GROUP);
	this.legacyCommittedOffsetsMetricGroup = consumerMetricGroup.addGroup(LEGACY_COMMITTED_OFFSETS_METRICS_GROUP);

	// figure out what we watermark mode we will be using
	this.watermarksPeriodic = watermarksPeriodic;
	this.watermarksPunctuated = watermarksPunctuated;

	if (watermarksPeriodic == null) {
		if (watermarksPunctuated == null) {
			// simple case, no watermarks involved
			timestampWatermarkMode = NO_TIMESTAMPS_WATERMARKS;
		} else {
			timestampWatermarkMode = PUNCTUATED_WATERMARKS;
		}
	} else {
		if (watermarksPunctuated == null) {
			timestampWatermarkMode = PERIODIC_WATERMARKS;
		} else {
			throw new IllegalArgumentException("Cannot have both periodic and punctuated watermarks");
		}
	}

	this.unassignedPartitionsQueue = new ClosableBlockingQueue<>();

	// initialize subscribed partition states with seed partitions
	this.subscribedPartitionStates = createPartitionStateHolders(
			seedPartitionsWithInitialOffsets,
			timestampWatermarkMode,
			watermarksPeriodic,
			watermarksPunctuated,
			userCodeClassLoader);

	// check that all seed partition states have a defined offset
	for (KafkaTopicPartitionState partitionState : subscribedPartitionStates) {
		if (!partitionState.isOffsetDefined()) {
			throw new IllegalArgumentException("The fetcher was assigned seed partitions with undefined initial offsets.");
		}
	}

	// all seed partitions are not assigned yet, so should be added to the unassigned partitions queue
	for (KafkaTopicPartitionState<KPH> partition : subscribedPartitionStates) {
		unassignedPartitionsQueue.add(partition);
	}

	// register metrics for the initial seed partitions
	if (useMetrics) {
		registerOffsetMetrics(consumerMetricGroup, subscribedPartitionStates);
	}

	// if we have periodic watermarks, kick off the interval scheduler
	if (timestampWatermarkMode == PERIODIC_WATERMARKS) {
		@SuppressWarnings("unchecked")
		PeriodicWatermarkEmitter periodicEmitter = new PeriodicWatermarkEmitter(
				subscribedPartitionStates,
				sourceContext,
				processingTimeProvider,
				autoWatermarkInterval);

		periodicEmitter.start();
	}
}
 

public Kafka09Fetcher(
		SourceContext<T> sourceContext,
		Map<KafkaTopicPartition, Long> assignedPartitionsWithInitialOffsets,
		SerializedValue<AssignerWithPeriodicWatermarks<T>> watermarksPeriodic,
		SerializedValue<AssignerWithPunctuatedWatermarks<T>> watermarksPunctuated,
		ProcessingTimeService processingTimeProvider,
		long autoWatermarkInterval,
		ClassLoader userCodeClassLoader,
		String taskNameWithSubtasks,
		KafkaDeserializationSchema<T> deserializer,
		Properties kafkaProperties,
		long pollTimeout,
		MetricGroup subtaskMetricGroup,
		MetricGroup consumerMetricGroup,
		boolean useMetrics,
		FlinkConnectorRateLimiter rateLimiter) throws Exception {
	super(
			sourceContext,
			assignedPartitionsWithInitialOffsets,
			watermarksPeriodic,
			watermarksPunctuated,
			processingTimeProvider,
			autoWatermarkInterval,
			userCodeClassLoader,
			consumerMetricGroup,
			useMetrics);

	this.deserializer = deserializer;
	this.handover = new Handover();

	this.consumerThread = new KafkaConsumerThread(
			LOG,
			handover,
			kafkaProperties,
			unassignedPartitionsQueue,
			createCallBridge(),
			getFetcherName() + " for " + taskNameWithSubtasks,
			pollTimeout,
			useMetrics,
			consumerMetricGroup,
			subtaskMetricGroup,
			rateLimiter);
}
 

protected AbstractFetcher(
		SourceContext<T> sourceContext,
		Map<KafkaTopicPartition, Long> seedPartitionsWithInitialOffsets,
		SerializedValue<AssignerWithPeriodicWatermarks<T>> watermarksPeriodic,
		SerializedValue<AssignerWithPunctuatedWatermarks<T>> watermarksPunctuated,
		ProcessingTimeService processingTimeProvider,
		long autoWatermarkInterval,
		ClassLoader userCodeClassLoader,
		MetricGroup consumerMetricGroup,
		boolean useMetrics) throws Exception {
	this.sourceContext = checkNotNull(sourceContext);
	this.checkpointLock = sourceContext.getCheckpointLock();
	this.userCodeClassLoader = checkNotNull(userCodeClassLoader);

	this.useMetrics = useMetrics;
	this.consumerMetricGroup = checkNotNull(consumerMetricGroup);
	this.legacyCurrentOffsetsMetricGroup = consumerMetricGroup.addGroup(LEGACY_CURRENT_OFFSETS_METRICS_GROUP);
	this.legacyCommittedOffsetsMetricGroup = consumerMetricGroup.addGroup(LEGACY_COMMITTED_OFFSETS_METRICS_GROUP);

	// figure out what we watermark mode we will be using
	this.watermarksPeriodic = watermarksPeriodic;
	this.watermarksPunctuated = watermarksPunctuated;

	if (watermarksPeriodic == null) {
		if (watermarksPunctuated == null) {
			// simple case, no watermarks involved
			timestampWatermarkMode = NO_TIMESTAMPS_WATERMARKS;
		} else {
			timestampWatermarkMode = PUNCTUATED_WATERMARKS;
		}
	} else {
		if (watermarksPunctuated == null) {
			timestampWatermarkMode = PERIODIC_WATERMARKS;
		} else {
			throw new IllegalArgumentException("Cannot have both periodic and punctuated watermarks");
		}
	}

	this.unassignedPartitionsQueue = new ClosableBlockingQueue<>();

	// initialize subscribed partition states with seed partitions
	this.subscribedPartitionStates = createPartitionStateHolders(
			seedPartitionsWithInitialOffsets,
			timestampWatermarkMode,
			watermarksPeriodic,
			watermarksPunctuated,
			userCodeClassLoader);

	// check that all seed partition states have a defined offset
	for (KafkaTopicPartitionState partitionState : subscribedPartitionStates) {
		if (!partitionState.isOffsetDefined()) {
			throw new IllegalArgumentException("The fetcher was assigned seed partitions with undefined initial offsets.");
		}
	}

	// all seed partitions are not assigned yet, so should be added to the unassigned partitions queue
	for (KafkaTopicPartitionState<KPH> partition : subscribedPartitionStates) {
		unassignedPartitionsQueue.add(partition);
	}

	// register metrics for the initial seed partitions
	if (useMetrics) {
		registerOffsetMetrics(consumerMetricGroup, subscribedPartitionStates);
	}

	// if we have periodic watermarks, kick off the interval scheduler
	if (timestampWatermarkMode == PERIODIC_WATERMARKS) {
		@SuppressWarnings("unchecked")
		PeriodicWatermarkEmitter periodicEmitter = new PeriodicWatermarkEmitter(
				subscribedPartitionStates,
				sourceContext,
				processingTimeProvider,
				autoWatermarkInterval);

		periodicEmitter.start();
	}
}
 

@Test
public void testConcurrentPartitionsDiscoveryAndLoopFetching() throws Exception {
	// test data
	final KafkaTopicPartition testPartition = new KafkaTopicPartition("test", 42);

	// ----- create the test fetcher -----

	@SuppressWarnings("unchecked")
	SourceContext<String> sourceContext = new TestSourceContext<>();
	Map<KafkaTopicPartition, Long> partitionsWithInitialOffsets =
		Collections.singletonMap(testPartition, KafkaTopicPartitionStateSentinel.GROUP_OFFSET);

	final OneShotLatch fetchLoopWaitLatch = new OneShotLatch();
	final OneShotLatch stateIterationBlockLatch = new OneShotLatch();

	final TestFetcher<String> fetcher = new TestFetcher<>(
		sourceContext,
		partitionsWithInitialOffsets,
		null, /* periodic assigner */
		null, /* punctuated assigner */
		new TestProcessingTimeService(),
		10,
		fetchLoopWaitLatch,
		stateIterationBlockLatch);

	// ----- run the fetcher -----

	final CheckedThread checkedThread = new CheckedThread() {
		@Override
		public void go() throws Exception {
			fetcher.runFetchLoop();
		}
	};
	checkedThread.start();

	// wait until state iteration begins before adding discovered partitions
	fetchLoopWaitLatch.await();
	fetcher.addDiscoveredPartitions(Collections.singletonList(testPartition));

	stateIterationBlockLatch.trigger();
	checkedThread.sync();
}
 

protected AbstractFetcher(
		SourceContext<T> sourceContext,
		Map<KafkaTopicPartition, Long> seedPartitionsWithInitialOffsets,
		SerializedValue<WatermarkStrategy<T>> watermarkStrategy,
		ProcessingTimeService processingTimeProvider,
		long autoWatermarkInterval,
		ClassLoader userCodeClassLoader,
		MetricGroup consumerMetricGroup,
		boolean useMetrics) throws Exception {
	this.sourceContext = checkNotNull(sourceContext);
	this.watermarkOutput = new SourceContextWatermarkOutputAdapter<>(sourceContext);
	this.watermarkOutputMultiplexer = new WatermarkOutputMultiplexer(watermarkOutput);
	this.checkpointLock = sourceContext.getCheckpointLock();
	this.userCodeClassLoader = checkNotNull(userCodeClassLoader);

	this.useMetrics = useMetrics;
	this.consumerMetricGroup = checkNotNull(consumerMetricGroup);
	this.legacyCurrentOffsetsMetricGroup = consumerMetricGroup.addGroup(LEGACY_CURRENT_OFFSETS_METRICS_GROUP);
	this.legacyCommittedOffsetsMetricGroup = consumerMetricGroup.addGroup(LEGACY_COMMITTED_OFFSETS_METRICS_GROUP);

	this.watermarkStrategy = watermarkStrategy;

	if (watermarkStrategy == null) {
		timestampWatermarkMode = NO_TIMESTAMPS_WATERMARKS;
	} else {
		timestampWatermarkMode = WITH_WATERMARK_GENERATOR;
	}

	this.unassignedPartitionsQueue = new ClosableBlockingQueue<>();

	// initialize subscribed partition states with seed partitions
	this.subscribedPartitionStates = createPartitionStateHolders(
			seedPartitionsWithInitialOffsets,
			timestampWatermarkMode,
			watermarkStrategy,
			userCodeClassLoader);

	// check that all seed partition states have a defined offset
	for (KafkaTopicPartitionState<?, ?> partitionState : subscribedPartitionStates) {
		if (!partitionState.isOffsetDefined()) {
			throw new IllegalArgumentException("The fetcher was assigned seed partitions with undefined initial offsets.");
		}
	}

	// all seed partitions are not assigned yet, so should be added to the unassigned partitions queue
	for (KafkaTopicPartitionState<T, KPH> partition : subscribedPartitionStates) {
		unassignedPartitionsQueue.add(partition);
	}

	// register metrics for the initial seed partitions
	if (useMetrics) {
		registerOffsetMetrics(consumerMetricGroup, subscribedPartitionStates);
	}

	// if we have periodic watermarks, kick off the interval scheduler
	if (timestampWatermarkMode == WITH_WATERMARK_GENERATOR && autoWatermarkInterval > 0) {
		PeriodicWatermarkEmitter<T, KPH> periodicEmitter = new PeriodicWatermarkEmitter<>(
				checkpointLock,
				subscribedPartitionStates,
				watermarkOutputMultiplexer,
				processingTimeProvider,
				autoWatermarkInterval);

		periodicEmitter.start();
	}
}
 

public SourceContextWatermarkOutputAdapter(SourceContext<T> sourceContext) {
	this.sourceContext = sourceContext;
}
 

default void withCheckpointLock(SourceContext<String> ctx)
throws Exception {}