Java Code Examples for org.apache.flink.streaming.connectors.kafka.internals.AbstractPartitionDiscoverer#WakeupException

The following examples show how to use org.apache.flink.streaming.connectors.kafka.internals.AbstractPartitionDiscoverer#WakeupException . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: KafkaPartitionDiscoverer.java From Flink-CEPplus with Apache License 2.0

6 votes

@Override
protected List<KafkaTopicPartition> getAllPartitionsForTopics(List<String> topics) throws AbstractPartitionDiscoverer.WakeupException {
	List<KafkaTopicPartition> partitions = new LinkedList<>();

	try {
		for (String topic : topics) {
			for (PartitionInfo partitionInfo : kafkaConsumer.partitionsFor(topic)) {
				partitions.add(new KafkaTopicPartition(partitionInfo.topic(), partitionInfo.partition()));
			}
		}
	} catch (org.apache.kafka.common.errors.WakeupException e) {
		// rethrow our own wakeup exception
		throw new AbstractPartitionDiscoverer.WakeupException();
	}

	return partitions;
}

Example 2

Source File: KafkaPartitionDiscoverer.java From Flink-CEPplus with Apache License 2.0

5 votes

@Override
protected List<String> getAllTopics() throws AbstractPartitionDiscoverer.WakeupException {
	try {
		return new ArrayList<>(kafkaConsumer.listTopics().keySet());
	} catch (org.apache.kafka.common.errors.WakeupException e) {
		// rethrow our own wakeup exception
		throw new AbstractPartitionDiscoverer.WakeupException();
	}
}

Example 3

Source File: KafkaPartitionDiscoverer.java From flink with Apache License 2.0

5 votes

@Override
protected List<String> getAllTopics() throws AbstractPartitionDiscoverer.WakeupException {
	try {
		return new ArrayList<>(kafkaConsumer.listTopics().keySet());
	} catch (org.apache.kafka.common.errors.WakeupException e) {
		// rethrow our own wakeup exception
		throw new AbstractPartitionDiscoverer.WakeupException();
	}
}

Example 4

Source File: KafkaPartitionDiscoverer.java From flink with Apache License 2.0

5 votes

@Override
protected List<String> getAllTopics() throws AbstractPartitionDiscoverer.WakeupException {
	try {
		return new ArrayList<>(kafkaConsumer.listTopics().keySet());
	} catch (org.apache.kafka.common.errors.WakeupException e) {
		// rethrow our own wakeup exception
		throw new AbstractPartitionDiscoverer.WakeupException();
	}
}

Example 5

Source File: FlinkKafkaConsumerBase.java From Flink-CEPplus with Apache License 2.0

4 votes

private void createAndStartDiscoveryLoop(AtomicReference<Exception> discoveryLoopErrorRef) {
	discoveryLoopThread = new Thread(() -> {
		try {
			// --------------------- partition discovery loop ---------------------

			// throughout the loop, we always eagerly check if we are still running before
			// performing the next operation, so that we can escape the loop as soon as possible

			while (running) {
				if (LOG.isDebugEnabled()) {
					LOG.debug("Consumer subtask {} is trying to discover new partitions ...", getRuntimeContext().getIndexOfThisSubtask());
				}

				final List<KafkaTopicPartition> discoveredPartitions;
				try {
					discoveredPartitions = partitionDiscoverer.discoverPartitions();
				} catch (AbstractPartitionDiscoverer.WakeupException | AbstractPartitionDiscoverer.ClosedException e) {
					// the partition discoverer may have been closed or woken up before or during the discovery;
					// this would only happen if the consumer was canceled; simply escape the loop
					break;
				}

				// no need to add the discovered partitions if we were closed during the meantime
				if (running && !discoveredPartitions.isEmpty()) {
					kafkaFetcher.addDiscoveredPartitions(discoveredPartitions);
				}

				// do not waste any time sleeping if we're not running anymore
				if (running && discoveryIntervalMillis != 0) {
					try {
						Thread.sleep(discoveryIntervalMillis);
					} catch (InterruptedException iex) {
						// may be interrupted if the consumer was canceled midway; simply escape the loop
						break;
					}
				}
			}
		} catch (Exception e) {
			discoveryLoopErrorRef.set(e);
		} finally {
			// calling cancel will also let the fetcher loop escape
			// (if not running, cancel() was already called)
			if (running) {
				cancel();
			}
		}
	}, "Kafka Partition Discovery for " + getRuntimeContext().getTaskNameWithSubtasks());

	discoveryLoopThread.start();
}

Example 6

Source File: FlinkKafkaConsumerBase.java From flink with Apache License 2.0

4 votes

private void createAndStartDiscoveryLoop(AtomicReference<Exception> discoveryLoopErrorRef) {
	discoveryLoopThread = new Thread(() -> {
		try {
			// --------------------- partition discovery loop ---------------------

			// throughout the loop, we always eagerly check if we are still running before
			// performing the next operation, so that we can escape the loop as soon as possible

			while (running) {
				if (LOG.isDebugEnabled()) {
					LOG.debug("Consumer subtask {} is trying to discover new partitions ...", getRuntimeContext().getIndexOfThisSubtask());
				}

				final List<KafkaTopicPartition> discoveredPartitions;
				try {
					discoveredPartitions = partitionDiscoverer.discoverPartitions();
				} catch (AbstractPartitionDiscoverer.WakeupException | AbstractPartitionDiscoverer.ClosedException e) {
					// the partition discoverer may have been closed or woken up before or during the discovery;
					// this would only happen if the consumer was canceled; simply escape the loop
					break;
				}

				// no need to add the discovered partitions if we were closed during the meantime
				if (running && !discoveredPartitions.isEmpty()) {
					kafkaFetcher.addDiscoveredPartitions(discoveredPartitions);
				}

				// do not waste any time sleeping if we're not running anymore
				if (running && discoveryIntervalMillis != 0) {
					try {
						Thread.sleep(discoveryIntervalMillis);
					} catch (InterruptedException iex) {
						// may be interrupted if the consumer was canceled midway; simply escape the loop
						break;
					}
				}
			}
		} catch (Exception e) {
			discoveryLoopErrorRef.set(e);
		} finally {
			// calling cancel will also let the fetcher loop escape
			// (if not running, cancel() was already called)
			if (running) {
				cancel();
			}
		}
	}, "Kafka Partition Discovery for " + getRuntimeContext().getTaskNameWithSubtasks());

	discoveryLoopThread.start();
}

Example 7

Source File: FlinkKafkaConsumerBase.java From flink with Apache License 2.0

4 votes

private void createAndStartDiscoveryLoop(AtomicReference<Exception> discoveryLoopErrorRef) {
	discoveryLoopThread = new Thread(() -> {
		try {
			// --------------------- partition discovery loop ---------------------

			// throughout the loop, we always eagerly check if we are still running before
			// performing the next operation, so that we can escape the loop as soon as possible

			while (running) {
				if (LOG.isDebugEnabled()) {
					LOG.debug("Consumer subtask {} is trying to discover new partitions ...", getRuntimeContext().getIndexOfThisSubtask());
				}

				final List<KafkaTopicPartition> discoveredPartitions;
				try {
					discoveredPartitions = partitionDiscoverer.discoverPartitions();
				} catch (AbstractPartitionDiscoverer.WakeupException | AbstractPartitionDiscoverer.ClosedException e) {
					// the partition discoverer may have been closed or woken up before or during the discovery;
					// this would only happen if the consumer was canceled; simply escape the loop
					break;
				}

				// no need to add the discovered partitions if we were closed during the meantime
				if (running && !discoveredPartitions.isEmpty()) {
					kafkaFetcher.addDiscoveredPartitions(discoveredPartitions);
				}

				// do not waste any time sleeping if we're not running anymore
				if (running && discoveryIntervalMillis != 0) {
					try {
						Thread.sleep(discoveryIntervalMillis);
					} catch (InterruptedException iex) {
						// may be interrupted if the consumer was canceled midway; simply escape the loop
						break;
					}
				}
			}
		} catch (Exception e) {
			discoveryLoopErrorRef.set(e);
		} finally {
			// calling cancel will also let the fetcher loop escape
			// (if not running, cancel() was already called)
			if (running) {
				cancel();
			}
		}
	}, "Kafka Partition Discovery for " + getRuntimeContext().getTaskNameWithSubtasks());

	discoveryLoopThread.start();
}