Java Code Examples for org.apache.kafka.clients.consumer.Consumer#partitionsFor()

The following examples show how to use org.apache.kafka.clients.consumer.Consumer#partitionsFor() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: KafkaValidationUtil09.java    From datacollector with Apache License 2.0 6 votes vote down vote up 
@Override
public int getPartitionCount(
    String metadataBrokerList,
    String topic,
    Map<String, Object> kafkaClientConfigs,
    int messageSendMaxRetries,
    long retryBackoffMs
) throws StageException {
  int partitionCount = -1;
  Consumer<String, String> kafkaConsumer = null;
  try {
    kafkaConsumer = createTopicMetadataClient(metadataBrokerList, kafkaClientConfigs);
    List<PartitionInfo> partitionInfoList = kafkaConsumer.partitionsFor(topic);
    if(partitionInfoList != null) {
      partitionCount = partitionInfoList.size();
    }
  } catch (KafkaException e) {
    LOG.error(KafkaErrors.KAFKA_41.getMessage(), topic, e.toString(), e);
    throw new StageException(KafkaErrors.KAFKA_41, topic, e.toString());
  } finally {
    if (kafkaConsumer != null) {
      kafkaConsumer.close();
    }
  }
  return partitionCount;
}
Example 2
Source File: MapR61StreamsValidationUtil11.java    From datacollector with Apache License 2.0 6 votes vote down vote up 
@Override
public int getPartitionCount(
    String metadataBrokerList,
    String topic,
    Map<String, Object> kafkaClientConfigs,
    int messageSendMaxRetries,
    long retryBackoffMs
) throws StageException {
  int partitionCount = -1;
  try {
    Consumer<String, String> kafkaConsumer = createTopicMetadataClient();
    List<PartitionInfo> partitionInfoList = kafkaConsumer.partitionsFor(topic);
    if(partitionInfoList != null) {
      partitionCount = partitionInfoList.size();
    }
  } catch (KafkaException e) {
    LOG.error(KafkaErrors.KAFKA_41.getMessage(), topic, e.toString(), e);
    throw new StageException(KafkaErrors.KAFKA_41, topic, e.toString(), e);
  }
  return partitionCount;
}
Example 3
Source File: MapRStreamsValidationUtil09.java    From datacollector with Apache License 2.0 6 votes vote down vote up 
@Override
public int getPartitionCount(
    String metadataBrokerList,
    String topic,
    Map<String, Object> kafkaClientConfigs,
    int messageSendMaxRetries,
    long retryBackoffMs
) throws StageException {
  int partitionCount = -1;
  try {
    Consumer<String, String> kafkaConsumer = createTopicMetadataClient();
    List<PartitionInfo> partitionInfoList = kafkaConsumer.partitionsFor(topic);
    if(partitionInfoList != null) {
      partitionCount = partitionInfoList.size();
    }
  } catch (KafkaException e) {
    LOG.error(KafkaErrors.KAFKA_41.getMessage(), topic, e.toString(), e);
    throw new StageException(KafkaErrors.KAFKA_41, topic, e.toString(), e);
  }
  return partitionCount;
}
Example 4
Source File: KafkaSimpleStreamingSource.java    From incubator-gobblin with Apache License 2.0 6 votes vote down vote up 
@Override
public List<WorkUnit> getWorkunits(SourceState state) {
  Config config = ConfigUtils.propertiesToConfig(state.getProperties());
  Consumer<String, byte[]> consumer = getKafkaConsumer(config);
  LOG.debug("Consumer is {}", consumer);
  String topic = ConfigUtils.getString(config, TOPIC_WHITELIST,
      StringUtils.EMPTY); // TODO: fix this to use the new API when KafkaWrapper is fixed
  List<WorkUnit> workUnits = new ArrayList<WorkUnit>();
  List<PartitionInfo> topicPartitions;
  topicPartitions = consumer.partitionsFor(topic);
  LOG.info("Partition count is {}", topicPartitions.size());
  for (PartitionInfo topicPartition : topicPartitions) {
    Extract extract = this.createExtract(DEFAULT_TABLE_TYPE, DEFAULT_NAMESPACE_NAME, topicPartition.topic());
    LOG.info("Partition info is {}", topicPartition);
    WorkUnit workUnit = WorkUnit.create(extract);
    setTopicNameInState(workUnit, topicPartition.topic());
    workUnit.setProp(ConfigurationKeys.EXTRACT_TABLE_NAME_KEY, topicPartition.topic());
    setPartitionId(workUnit, topicPartition.partition());
    workUnits.add(workUnit);
  }
  return workUnits;
}
Example 5
Source File: SubscriptionManager.java    From kafka-pubsub-emulator with Apache License 2.0 5 votes vote down vote up 
/**
 * Initializes and returns a List of Consumers that are manually assigned to specific
 * TopicPartitions. We choose to use manual assignment to avoid the timeout, blocking, and
 * heartbeats required when using dynamic subscriptions.
 *
 * @return List of Consumers assigned to partitions from the topic
 */
@Override
protected void startUp() {
  // Create first consumer and determine # partitions in topic
  Consumer<String, ByteBuffer> first = kafkaClientFactory.createConsumer(subscription.getName());
  List<PartitionInfo> partitionInfo = first.partitionsFor(kafkaTopicName);

  int totalConsumers = Math.min(partitionInfo.size(), consumerExecutors);
  logger.atFine().log(
      "Assigning %d KafkaConsumers to %d partitions of Kafka topic %s for Subscription %s",
      totalConsumers, partitionInfo.size(), kafkaTopicName, subscription.getName());
  for (int i = 0; i < totalConsumers; i++) {
    Consumer<String, ByteBuffer> consumer =
        i == 0 ? first : kafkaClientFactory.createConsumer(subscription.getName());
    int consumerIndex = i;
    Set<TopicPartition> partitionSet =
        partitionInfo
            .stream()
            .filter(p -> p.partition() % totalConsumers == consumerIndex)
            .map(p -> new TopicPartition(kafkaTopicName, p.partition()))
            .collect(Collectors.toSet());
    consumer.assign(partitionSet);
    Map<TopicPartition, Long> endOffsets = consumer.endOffsets(partitionSet);
    for (TopicPartition tp : partitionSet) {
      committedOffsets.put(tp, consumer.committed(tp));
      logger.atFine().log(
          "%s assigned KafkaConsumer %d to %s:%d (End: %d, Committed %d)",
          subscription.getName(),
          consumerIndex,
          tp.topic(),
          tp.partition(),
          endOffsets.get(tp),
          Optional.ofNullable(consumer.committed(tp)).map(OffsetAndMetadata::offset).orElse(0L));
    }
    kafkaConsumers.add(new SynchronizedConsumer(consumer));
  }
}
Example 6
Source File: MapR52StreamsValidationUtil09.java    From datacollector with Apache License 2.0 4 votes vote down vote up 
@Override
public boolean validateTopicExistence(
  Stage.Context context,
  String groupName,
  String configName,
  List<HostAndPort> kafkaBrokers,
  String metadataBrokerList,
  String topic,
  Map<String, Object> kafkaClientConfigs,
  List<Stage.ConfigIssue> issues,
  boolean producer
) {
  boolean valid = true;
  if(topic == null || topic.isEmpty()) {
    issues.add(context.createConfigIssue(groupName, configName, KafkaErrors.KAFKA_05));
    valid = false;
  } else {
    List<PartitionInfo> partitionInfos;
    try {
      // Use KafkaConsumer to check the topic existance
      // Using KafkaProducer causes creating a topic if not exist, even if runtime topic resolution is set to false.
      Consumer<String, String> kafkaConsumer = createTopicMetadataClient();
      partitionInfos = kafkaConsumer.partitionsFor(topic);
      if (null == partitionInfos || partitionInfos.isEmpty()) {
        issues.add(
            context.createConfigIssue(
                groupName,
                KAFKA_CONFIG_BEAN_PREFIX + "topic",
                MapRStreamsErrors.MAPRSTREAMS_02,
                topic
            )
        );
        valid = false;
      }
    } catch (KafkaException e) {
      LOG.error(MapRStreamsErrors.MAPRSTREAMS_01.getMessage(), topic, e.toString(), e);
      issues.add(
          context.createConfigIssue(
              groupName,
              configName,
              MapRStreamsErrors.MAPRSTREAMS_01,
              topic,
              e.getMessage()
          )
      );
      valid = false;
    }
  }
  return valid;
}
Example 7
Source File: KafkaValidationUtil09.java    From datacollector with Apache License 2.0 4 votes vote down vote up 
@Override
public boolean validateTopicExistence(
  Stage.Context context,
  String groupName,
  String configName,
  List<HostAndPort> kafkaBrokers,
  String metadataBrokerList,
  String topic,
  Map<String, Object> kafkaClientConfigs,
  List<Stage.ConfigIssue> issues,
  boolean producer
) {
  boolean valid = true;
  if(topic == null || topic.isEmpty()) {
    issues.add(context.createConfigIssue(groupName, configName, KafkaErrors.KAFKA_05));
    valid = false;
  } else {
    List<PartitionInfo> partitionInfos;
    // Use Consumer to check if topic exists.
    // Using producer causes unintentionally creating a topic if not exist.
    Consumer<String, String> kafkaConsumer = null;
    try {
        kafkaConsumer = createTopicMetadataClient(
            metadataBrokerList,
            kafkaClientConfigs
        );
        partitionInfos = kafkaConsumer.partitionsFor(topic);
      if (null == partitionInfos || partitionInfos.isEmpty()) {
        issues.add(
            context.createConfigIssue(
                groupName,
                KAFKA_CONFIG_BEAN_PREFIX + "topic",
                KafkaErrors.KAFKA_03,
                topic,
                metadataBrokerList
            )
        );
        valid = false;
      }
    } catch (KafkaException e) {
      LOG.error(KafkaErrors.KAFKA_68.getMessage(), topic, metadataBrokerList, e.toString(), e);
      issues.add(context.createConfigIssue(groupName, configName, KafkaErrors.KAFKA_68, topic, metadataBrokerList, e.toString()));
      valid = false;
    } finally {
      if (kafkaConsumer != null) {
        kafkaConsumer.close();
      }
    }
  }
  return valid;
}
Example 8
Source File: MapR61StreamsValidationUtil11.java    From datacollector with Apache License 2.0 4 votes vote down vote up 
@Override
public boolean validateTopicExistence(
  Stage.Context context,
  String groupName,
  String configName,
  List<HostAndPort> kafkaBrokers,
  String metadataBrokerList,
  String topic,
  Map<String, Object> kafkaClientConfigs,
  List<Stage.ConfigIssue> issues,
  boolean producer
) {
  boolean valid = true;
  if(topic == null || topic.isEmpty()) {
    issues.add(context.createConfigIssue(groupName, configName, KafkaErrors.KAFKA_05));
    valid = false;
  } else {
    List<PartitionInfo> partitionInfos;
    try {
      // Use KafkaConsumer to check the topic existance
      // Using Producer causes creating a topic if not exist, even if runtime topic resolution is set to false.
      Consumer<String, String> kafkaConsumer = createTopicMetadataClient();
      partitionInfos = kafkaConsumer.partitionsFor(topic);
      if (null == partitionInfos || partitionInfos.isEmpty()) {
        issues.add(
            context.createConfigIssue(
                groupName,
                KAFKA_CONFIG_BEAN_PREFIX + "topic",
                MapRStreamsErrors.MAPRSTREAMS_02,
                topic
            )
        );
        valid = false;
      }
    } catch (KafkaException e) {
      LOG.error(MapRStreamsErrors.MAPRSTREAMS_01.getMessage(), topic, e.toString(), e);
      issues.add(
          context.createConfigIssue(
              groupName,
              configName,
              MapRStreamsErrors.MAPRSTREAMS_01,
              topic,
              e.getMessage()
          )
      );
      valid = false;
    }
  }
  return valid;
}
Example 9
Source File: MapRStreamsValidationUtil09.java    From datacollector with Apache License 2.0 4 votes vote down vote up 
@Override
public boolean validateTopicExistence(
  Stage.Context context,
  String groupName,
  String configName,
  List<HostAndPort> kafkaBrokers,
  String metadataBrokerList,
  String topic,
  Map<String, Object> kafkaClientConfigs,
  List<Stage.ConfigIssue> issues,
  boolean producer
) {
  boolean valid = true;
  if(topic == null || topic.isEmpty()) {
    issues.add(context.createConfigIssue(groupName, configName, KafkaErrors.KAFKA_05));
    valid = false;
  } else {
    List<PartitionInfo> partitionInfos;
    try {
      if (producer) {
        Producer<String, String> kafkaProducer = createProducerTopicMetadataClient(kafkaClientConfigs);
        partitionInfos = kafkaProducer.partitionsFor(topic);
      } else {
        Consumer<String, String> kafkaConsumer = createTopicMetadataClient();
        partitionInfos = kafkaConsumer.partitionsFor(topic);
      }
      if (null == partitionInfos || partitionInfos.isEmpty()) {
        issues.add(
            context.createConfigIssue(
                groupName,
                KAFKA_CONFIG_BEAN_PREFIX + "topic",
                MapRStreamsErrors.MAPRSTREAMS_02,
                topic
            )
        );
        valid = false;
      }
    } catch (KafkaException e) {
      LOG.error(MapRStreamsErrors.MAPRSTREAMS_01.getMessage(), topic, e.toString(), e);
      issues.add(
          context.createConfigIssue(
              groupName,
              configName,
              MapRStreamsErrors.MAPRSTREAMS_01,
              topic,
              e.getMessage()
          )
      );
      valid = false;
    }
  }
  return valid;
}