Java Code Examples for org.apache.kafka.clients.producer.KafkaProducer#partitionsFor()

The following examples show how to use org.apache.kafka.clients.producer.KafkaProducer#partitionsFor() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: FlinkKafkaProducerBase.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
protected static int[] getPartitionsByTopic(String topic, KafkaProducer<byte[], byte[]> producer) {
	// the fetched list is immutable, so we're creating a mutable copy in order to sort it
	List<PartitionInfo> partitionsList = new ArrayList<>(producer.partitionsFor(topic));

	// sort the partitions by partition id to make sure the fetched partition list is the same across subtasks
	Collections.sort(partitionsList, new Comparator<PartitionInfo>() {
		@Override
		public int compare(PartitionInfo o1, PartitionInfo o2) {
			return Integer.compare(o1.partition(), o2.partition());
		}
	});

	int[] partitions = new int[partitionsList.size()];
	for (int i = 0; i < partitions.length; i++) {
		partitions[i] = partitionsList.get(i).partition();
	}

	return partitions;
}
Example 2
Source File: FlinkKafkaProducerBase.java    From flink with Apache License 2.0 6 votes vote down vote up 
protected static int[] getPartitionsByTopic(String topic, KafkaProducer<byte[], byte[]> producer) {
	// the fetched list is immutable, so we're creating a mutable copy in order to sort it
	List<PartitionInfo> partitionsList = new ArrayList<>(producer.partitionsFor(topic));

	// sort the partitions by partition id to make sure the fetched partition list is the same across subtasks
	Collections.sort(partitionsList, new Comparator<PartitionInfo>() {
		@Override
		public int compare(PartitionInfo o1, PartitionInfo o2) {
			return Integer.compare(o1.partition(), o2.partition());
		}
	});

	int[] partitions = new int[partitionsList.size()];
	for (int i = 0; i < partitions.length; i++) {
		partitions[i] = partitionsList.get(i).partition();
	}

	return partitions;
}
Example 3
Source File: FlinkKafkaProducerBase.java    From flink with Apache License 2.0 6 votes vote down vote up 
protected static int[] getPartitionsByTopic(String topic, KafkaProducer<byte[], byte[]> producer) {
	// the fetched list is immutable, so we're creating a mutable copy in order to sort it
	List<PartitionInfo> partitionsList = new ArrayList<>(producer.partitionsFor(topic));

	// sort the partitions by partition id to make sure the fetched partition list is the same across subtasks
	Collections.sort(partitionsList, new Comparator<PartitionInfo>() {
		@Override
		public int compare(PartitionInfo o1, PartitionInfo o2) {
			return Integer.compare(o1.partition(), o2.partition());
		}
	});

	int[] partitions = new int[partitionsList.size()];
	for (int i = 0; i < partitions.length; i++) {
		partitions[i] = partitionsList.get(i).partition();
	}

	return partitions;
}
Example 4
Source File: KafkaInstrumentationHelperImpl.java    From apm-agent-java with Apache License 2.0 5 votes vote down vote up 
@Override
public void onSendEnd(Span span, ProducerRecord producerRecord, KafkaProducer kafkaProducer, @Nullable Throwable throwable) {

    // Topic address collection is normally very fast, as it uses cached cluster state information. However,
    // when the cluster metadata is required to be updated, its query may block for a short period. In
    // addition, the discovery operation allocates two objects. Therefore, we have the ability to turn it off.
    if (messagingConfiguration.shouldCollectQueueAddress()) {
        try {
            // Better get the destination now, as if the partition's leader was replaced, it may be reflected at
            // this point.
            @SuppressWarnings("unchecked") List<PartitionInfo> partitions = kafkaProducer.partitionsFor(producerRecord.topic());
            Integer partition = producerRecord.partition();
            PartitionInfo partitionInfo = null;
            if (partition != null) {
                partitionInfo = partitions.get(partition);
            } else if (!partitions.isEmpty()) {
                // probably not a partitioned topic, so look for the singe entry
                partitionInfo = partitions.get(0);
            }
            if (partitionInfo != null) {
                // Records are always sent to the leader of a partition and then synced with replicas internally by
                // the broker.
                Node leader = partitionInfo.leader();
                if (leader != null) {
                    span.getContext().getDestination().withAddress(leader.host()).withPort(leader.port());
                }
            }
        } catch (Exception e) {
            logger.error("Failed to get Kafka producer's destination", e);
        }
    }

    span.captureException(throwable);

    // Not ending here- ending in the callback
    span.deactivate();
}