kafka.common.TopicAndPartition Java Examples

private int readNext() {
   FetchRequestBuilder requestBuilder =
         new FetchRequestBuilder()
            .clientId(clientId);
   for(Map.Entry<TopicAndPartition, Long> offset: offsets.entrySet()) {
      if(offset.getValue() == null) {
         logger.warn("Invalid offset for topic: [{}] partition: [{}]", offset.getKey().topic(), offset.getKey().partition());
      }
      else {
         requestBuilder.addFetch(offset.getKey().topic(), offset.getKey().partition(), offset.getValue(), fetchSize);
      }
   }
   FetchRequest request = requestBuilder.build();
   FetchResponse response = getKafkaConsumer().fetch(request);
   // FIXME handle errors / leader rebalances hear
   return dispatch(response);
}
 

private void commitAll()
        throws Exception
{
    Map<TopicAndPartition, Long> m = new HashMap<>();
    KafkaPartitionOffset osr = commitQueue.poll();
    while (null != osr) {
        TopicAndPartition tp = osr.getTopicPartition();
        Long x = m.get(tp);
        long offset = (null == x) ? osr.getOffset() : Math.max(x, osr.getOffset());
        m.put(tp, offset);
        osr = commitQueue.poll();
    }
    if (!m.isEmpty()) {
        commitKafkaOffsets(m);
        //consumer.commitAsync(m, commitCallback.get)
    }
}
 

private static long getLastOffset(SimpleConsumer consumer, String topic, int partition,
                                  long whichTime) {
    TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partition);
    Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
    requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(whichTime, 1));
    kafka.javaapi.OffsetRequest request = new kafka.javaapi.OffsetRequest(requestInfo,
        kafka.api.OffsetRequest.CurrentVersion(), CLIENT_ID);
    OffsetResponse response = consumer.getOffsetsBefore(request);

    if (response.hasError()) {
        System.out.println("Error fetching data Offset Data the Broker. Reason: "
                           + response.errorCode(topic, partition));
        return 0;
    }
    long[] offsets = response.offsets(topic, partition);
    return offsets[0];
}
 

public static long getOffset(SimpleConsumer consumer, String topic, int partition,
                             long whichTime) throws IOException {
  String errMsg = null;
  Exception lastEx = null;
  for (int i = 0; i < FETCH_RETRIES; i++) {
    TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partition);
    try {
      long offset = consumer.earliestOrLatestOffset(topicAndPartition, whichTime,
          Request.DebuggingConsumerId());
      return offset;
    } catch (RuntimeException e) {
      lastEx = e;
      errMsg = "Failed to getting offset for topic: " + topic + " partition: " + partition
          + " host: " + consumer.host();
      LOG.warn(errMsg, e);
      try {
        Thread.sleep((long) (Math.random() * 3000));
      } catch (InterruptedException ex) {
        LOG.warn("Unexpected interruption", ex);
      }
    }
  }
  throw new IOException(errMsg, lastEx);
}
 

private scala.collection.Map<TopicAndPartition, Seq<Object>> getAssignmentPlan(
    Map<TopicPartition, Integer[]> replicasMap) {
  scala.collection.mutable.HashMap<TopicAndPartition, Seq<Object>> result =
      new scala.collection.mutable.HashMap<>();

  for (Map.Entry<TopicPartition, Integer[]> entry : replicasMap.entrySet()) {
    TopicPartition tp = entry.getKey();
    TopicAndPartition tap = new TopicAndPartition(tp.topic(), tp.partition());
    List<Object> objs = Arrays.asList(entry.getValue()).stream()
        .map(val -> (Object) val).collect(Collectors.toList());
    Seq<Object> replicas = JavaConverters.asScalaBuffer(objs).seq();
    result.put(tap, replicas);
  }

  assert replicasMap.size() == result.size();
  LOG.debug("replicaMap.size = {}, result.size = {}", replicasMap.size(), result.size());
  return result;
}
 

KafkaLeaderReader(
      KafkaConsumerConfig config,
      KafkaConsumerFactory handlerFactory,
      Supplier<SimpleConsumer> supplier,
      List<TopicAndPartition> partitions
) {
   this.topic = config.getTopics();
   this.clientId = config.getClientId();
   this.scanFetchSize = config.getScanFetchSize();
   this.fetchSize = config.getFetchSize();
   this.emptyReadWait = config.getEmptyWaitTime();
   this.emptyReadMax = config.getEmptyWaitMaxTime();
   this.emptyExitTime = config.getEmptyExitTime();

   this.handlers = new HashMap<>();
   this.handlerFactory = handlerFactory;
   this.consumerFactory = supplier;
   this.partitions = ImmutableList.copyOf(partitions);

   this.offsets = new LinkedHashMap<>(); // we iterate this a lot
}
 

private SimpleConsumerThread<T> createAndStartSimpleConsumerThread(
		List<KafkaTopicPartitionState<TopicAndPartition>> seedPartitions,
		Node leader,
		ExceptionProxy errorHandler) throws IOException, ClassNotFoundException {
	// each thread needs its own copy of the deserializer, because the deserializer is
	// not necessarily thread safe
	final KafkaDeserializationSchema<T> clonedDeserializer =
			InstantiationUtil.clone(deserializer, runtimeContext.getUserCodeClassLoader());

	// seed thread with list of fetch partitions (otherwise it would shut down immediately again
	SimpleConsumerThread<T> brokerThread = new SimpleConsumerThread<>(
			this, errorHandler, kafkaConfig, leader, seedPartitions, unassignedPartitionsQueue,
			clonedDeserializer, invalidOffsetBehavior);

	brokerThread.setName(String.format("SimpleConsumer - %s - broker-%s (%s:%d)",
			runtimeContext.getTaskName(), leader.id(), leader.host(), leader.port()));
	brokerThread.setDaemon(true);
	brokerThread.start();

	LOG.info("Starting thread {}", brokerThread.getName());
	return brokerThread;
}
 

@Override
public synchronized long getEarliestOffset(String topic, int partition) {

    if (checkConsumer(topic, partition)) {

        TopicAndPartition topicAndPartition = new TopicAndPartition(topic,
                partition);
        Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
        requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(
                kafka.api.OffsetRequest.EarliestTime(), 1));
        kafka.javaapi.OffsetRequest request = new kafka.javaapi.OffsetRequest(
                requestInfo, kafka.api.OffsetRequest.CurrentVersion(),
                pool.getClientId());
        OffsetResponse response = consumer.get().getOffsetsBefore(request);

        if (response.hasError()) {
            logger.error("Error fetching data Offset Data the Broker. Reason: "
                    + response.errorCode(topic, partition));
            return 0;
        }
        long[] offsets = response.offsets(topic, partition);
        return offsets[0];
    }

    return -1;
}
 

public static long getLastOffset(SimpleConsumer consumer, String topic, int partition, long whichTime,
      String clientName) {
	TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partition);
	Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
	requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(whichTime, 1));
	kafka.javaapi.OffsetRequest request = new kafka.javaapi.OffsetRequest(requestInfo,
	      kafka.api.OffsetRequest.CurrentVersion(), clientName);
	OffsetResponse response = consumer.getOffsetsBefore(request);

	if (response.hasError()) {
		System.out.println("Error fetching data Offset Data the Broker. Reason: "
		      + response.errorCode(topic, partition));
		return 0;
	}
	long[] offsets = response.offsets(topic, partition);
	return offsets[0];
}
 

/**
 * @param consumer
 * @param topic
 * @param partition
 * @param whichTime
 * @param clientName
 * @return 0 if consumer is null at this time
 */
public static long getLastOffset(SimpleConsumer consumer, String topic, int partition, long whichTime, String clientName)
{
  if (consumer == null) {
    return 0;
  }
  TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partition);
  Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
  requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(whichTime, 1));
  OffsetRequest request = new OffsetRequest(requestInfo, kafka.api.OffsetRequest.CurrentVersion(), clientName);
  OffsetResponse response = consumer.getOffsetsBefore(request);

  if (response.hasError()) {
    logger.error("Error fetching data Offset Data the Broker. Reason: " + response.errorCode(topic, partition));
    return 0;
  }
  long[] offsets = response.offsets(topic, partition);
  return offsets[0];
}
 

private long getEarliestOffsetFromKafka(String topic, int partition, long startTime) {
  LOG.info("getEarliestOffsetFromKafka.");
  TopicAndPartition tp = new TopicAndPartition(topic, partition);

  Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo =
      new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();

  requestInfo.put(tp, new PartitionOffsetRequestInfo(startTime, 1));

  OffsetRequest req = new OffsetRequest(
      requestInfo, kafka.api.OffsetRequest.CurrentVersion(), getClientName());

  OffsetResponse resp = consumer.getOffsetsBefore(req);

  if (resp.hasError()) {
    LOG.error("error when fetching offset: " + resp.errorCode(topic, partition)); //xxx
    return 0;
  }
  LOG.info("Earliest offset " + resp.offsets(topic, partition)[0]);
  return resp.offsets(topic, partition)[0];
}
 

/**
 * 请求取topic偏移
 *
 * @param broker
 * @param topic
 * @param brokerPartitions
 * @return
 */
public static OffsetResponse sendOffsetRequest(Broker broker, Topic topic,
                                               List<Partition> brokerPartitions, long time) {

    PartitionOffsetRequestInfo requestInfo = new PartitionOffsetRequestInfo(time, 1);

    final OffsetRequest offsetRequest = new OffsetRequest(
            brokerPartitions.stream()
                    .collect(Collectors.toMap(
                            partition -> new TopicAndPartition(topic.getName(), partition.getId()),
                            partition -> requestInfo)), (short) 0, "kafkaMonitor");

    logger.debug("Sending offset request: {}", offsetRequest);
    if (broker != null) {
        BlockingChannel channel = getChannel(broker);
        channel.send(offsetRequest.underlying());
        final kafka.api.OffsetResponse underlyingResponse = kafka.api.OffsetResponse.readFrom(channel.receive().payload());
        channel.disconnect();
        return new OffsetResponse(underlyingResponse);
    } else {
        return null;
    }

}
 

private long getLatestOffset(SimpleConsumer consumer, TopicAndPartition topicAndPartition) {
  Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<>();
  requestInfo.put(topicAndPartition,
      new PartitionOffsetRequestInfo(kafka.api.OffsetRequest.LatestTime(), 1));
  kafka.javaapi.OffsetRequest request = new kafka.javaapi.OffsetRequest(requestInfo,
      kafka.api.OffsetRequest.CurrentVersion(), consumer.clientId());
  OffsetResponse response = consumer.getOffsetsBefore(request);

  if (response.hasError()) {
    logger.warn("Failed to fetch offset for {} due to {}", topicAndPartition,
        response.errorCode(topicAndPartition.topic(), topicAndPartition.partition()));
    return -1;
  }

  long[] offsets = response.offsets(topicAndPartition.topic(), topicAndPartition.partition());
  return offsets[0];
}
 

@Override
public synchronized long getLatestOffset(String topic, int partition) {

    if (checkConsumer(topic, partition)) {

        TopicAndPartition topicAndPartition = new TopicAndPartition(topic,
                partition);
        Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
        requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(
                kafka.api.OffsetRequest.LatestTime(), 1));
        kafka.javaapi.OffsetRequest request = new kafka.javaapi.OffsetRequest(
                requestInfo, kafka.api.OffsetRequest.CurrentVersion(),
                pool.getClientId());
        OffsetResponse response = consumer.get().getOffsetsBefore(request);

        if (response.hasError()) {
            logger.error("Error fetching data Offset Data the Broker. Reason: "
                    + response.errorCode(topic, partition));
            return 0;
        }
        long[] offsets = response.offsets(topic, partition);
        return offsets[0];
    }
    return -1;
}
 

public long getOffset(String topic, int partition, long startOffsetTime) {
    SimpleConsumer simpleConsumer = findLeaderConsumer(partition);

    if (simpleConsumer == null) {
        LOG.error("Error consumer is null get offset from partition:" + partition);
        return -1;
    }

    TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partition);
    Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
    requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(startOffsetTime, 1));
    OffsetRequest request = new OffsetRequest(requestInfo, kafka.api.OffsetRequest.CurrentVersion(), simpleConsumer.clientId());

    long[] offsets = simpleConsumer.getOffsetsBefore(request).offsets(topic, partition);
    if (offsets.length > 0) {
        return offsets[0];
    } else {
        return NO_OFFSET;
    }
}
 

@Override
public long getEarliestOffset() {
  if (this.earliestOffset == -2 && uri != null) {
    // TODO : Make the hardcoded paramters configurable
    SimpleConsumer consumer = new SimpleConsumer(uri.getHost(), uri.getPort(), 60000,
        1024 * 1024, "hadoop-etl");
    Map<TopicAndPartition, PartitionOffsetRequestInfo> offsetInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
    offsetInfo.put(new TopicAndPartition(topic, partition), new PartitionOffsetRequestInfo(
        kafka.api.OffsetRequest.EarliestTime(), 1));
    OffsetResponse response = consumer
        .getOffsetsBefore(new OffsetRequest(offsetInfo, kafka.api.OffsetRequest
            .CurrentVersion(), "hadoop-etl"));
    long[] endOffset = response.offsets(topic, partition);
    consumer.close();
    this.earliestOffset = endOffset[0];
    return endOffset[0];
  } else {
    return this.earliestOffset;
  }
}
 

@Override
public long getLastOffset(long time) {
  SimpleConsumer consumer = new SimpleConsumer(uri.getHost(), uri.getPort(), 60000,
      1024 * 1024, "hadoop-etl");
  Map<TopicAndPartition, PartitionOffsetRequestInfo> offsetInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
  offsetInfo.put(new TopicAndPartition(topic, partition), new PartitionOffsetRequestInfo(
      time, 1));
  OffsetResponse response = consumer.getOffsetsBefore(new OffsetRequest(offsetInfo,
      kafka.api.OffsetRequest.CurrentVersion(),"hadoop-etl"));
  long[] endOffset = response.offsets(topic, partition);
  consumer.close();
  if(endOffset.length == 0)
  {
    log.info("The exception is thrown because the latest offset retunred zero for topic : " + topic + " and partition " + partition);
  }
  this.latestOffset = endOffset[0];
  return endOffset[0];
}
 

private long getLastOffset(SimpleConsumer consumer, String topic, int partition,
                                 long whichTime, String clientName) throws StageException {
  try {
    TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partition);
    Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<>();
    requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(whichTime, 1));
    kafka.javaapi.OffsetRequest request = new kafka.javaapi.OffsetRequest(
      requestInfo, kafka.api.OffsetRequest.CurrentVersion(), clientName);
    OffsetResponse response = consumer.getOffsetsBefore(request);

    if (response.hasError()) {
      LOG.error(KafkaErrors.KAFKA_22.getMessage(), consumer.host() + ":" + consumer.port(),
        response.errorCode(topic, partition));
      return 0;
    }
    long[] offsets = response.offsets(topic, partition);
    return offsets[0];
  } catch (Exception e) {
    LOG.error(KafkaErrors.KAFKA_30.getMessage(), e.toString(), e);
    throw new StageException(KafkaErrors.KAFKA_30, e.toString(), e);
  }
}
 

public PulsarOffsetCommitRequest(String groupId, Map<TopicAndPartition, PulsarOffsetMetadataAndError> requestInfo,
        short versionId, int correlationId, String clientId) {
    super(groupId, Collections.emptyMap(), versionId, correlationId, clientId);
    this.groupId = groupId;
    for (Entry<TopicAndPartition, PulsarOffsetMetadataAndError> topicOffset : requestInfo.entrySet()) {
        String topicName = PulsarKafkaSimpleConsumer.getTopicName(topicOffset.getKey());
        OffsetMetadataAndError offsetMetadata = topicOffset.getValue();
        MessageId msgId = null;
        if (offsetMetadata instanceof PulsarOffsetMetadataAndError) {
            msgId = ((PulsarOffsetMetadataAndError) offsetMetadata).getMessageId();
        }
        msgId = msgId == null ? MessageIdUtils.getMessageId(topicOffset.getValue().offset()) : msgId;
        topicOffsetMap.put(topicName, msgId);
    }
}
 

private long getOffset(KafkaPartition partition, Map<TopicAndPartition, PartitionOffsetRequestInfo> offsetRequestInfo)
    throws KafkaOffsetRetrievalFailureException {
  SimpleConsumer consumer = this.getSimpleConsumer(partition.getLeader().getHostAndPort());
  for (int i = 0; i < this.fetchOffsetRetries; i++) {
    try {
      OffsetResponse offsetResponse =
          consumer.getOffsetsBefore(new OffsetRequest(offsetRequestInfo, kafka.api.OffsetRequest.CurrentVersion(),
              this.clientName));
      if (offsetResponse.hasError()) {
        throw new RuntimeException("offsetReponse has error: "
            + offsetResponse.errorCode(partition.getTopicName(), partition.getId()));
      }
      return offsetResponse.offsets(partition.getTopicName(), partition.getId())[0];
    } catch (Exception e) {
      log.warn(String.format("Fetching offset for partition %s has failed %d time(s). Reason: %s", partition, i + 1,
          e));
      if (i < this.fetchOffsetRetries - 1) {
        try {
          Thread.sleep((long) ((i + Math.random()) * 1000));
        } catch (InterruptedException e2) {
          log.error("Caught interrupted exception between retries of getting latest offsets. " + e2);
        }
      }
    }
  }
  throw new KafkaOffsetRetrievalFailureException(String.format("Fetching offset for partition %s has failed.",
      partition));
}
 

public long getLastOffset() throws InterruptedException {
    OffsetResponse response = null;
    Broker previousLeader = leaderBroker;
    while (true) {
        TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partitionId);
        Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
        requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(kafka.api.OffsetRequest.LatestTime(), 1));
        kafka.javaapi.OffsetRequest request = new kafka.javaapi.OffsetRequest(
                requestInfo, kafka.api.OffsetRequest.CurrentVersion(), clientId);

        ensureConsumer(previousLeader);
        try {
            response = consumer.getOffsetsBefore(request);
        } catch (Exception e) {
            // e could be an instance of ClosedByInterruptException as SimpleConsumer.fetch uses nio
            if (Thread.interrupted()) {
                logger.info("catch exception of {} with interrupted in getLastOffset for {} - {}",
                        e.getClass().getName(), topic, partitionId);

                throw new InterruptedException();
            }
            logger.warn("caughte exception in getLastOffset {} - {}", topic, partitionId, e);
            response = null;
        }
        if (response == null || response.hasError()) {
            short errorCode = response != null ? response.errorCode(topic, partitionId) : ErrorMapping.UnknownCode();

            logger.warn("Error fetching data Offset for {} - {}, the Broker. Reason: {}",
                    topic, partitionId, errorCode);

            stopConsumer();
            previousLeader = leaderBroker;
            leaderBroker = null;
            continue;
        }
        break;
    }
    long[] offsets = response.offsets(topic, partitionId);
    return offsets[offsets.length - 1];
}
 

@Override
public long getEarliestOffset(KafkaPartition partition) throws KafkaOffsetRetrievalFailureException {
  Map<TopicAndPartition, PartitionOffsetRequestInfo> offsetRequestInfo =
      Collections.singletonMap(new TopicAndPartition(partition.getTopicName(), partition.getId()),
          new PartitionOffsetRequestInfo(kafka.api.OffsetRequest.EarliestTime(), 1));
  return getOffset(partition, offsetRequestInfo);
}
 

@Override
public ByteBufferMessageSet messageSet(String topic, int partition) {
  if (errorMap.containsKey(new TopicAndPartition(topic, partition))) {
    throw new IllegalArgumentException();
  } else {
    // TODO Maybe generate dummy messages here?
    return new ByteBufferMessageSet(Collections.<Message>emptyList());
  }
}
 

private OffsetResponse sendOffsetRequest(Integer brokerId, TopicVO topic,
                                         PartitionOffsetRequestInfo requestInfo,
                                         List<TopicPartitionVO> brokerPartitions)
{
   final OffsetRequest offsetRequest = new OffsetRequest(
      brokerPartitions.stream()
         .collect(Collectors.toMap(
            partition -> new TopicAndPartition(topic.getName(), partition.getId()),
            partition -> requestInfo)),
      (short) 0, clientId());

   LOG.debug("Sending offset request: {}", offsetRequest);

   return retryTemplate.execute(
      context ->
         brokerChannel(brokerId)
            .execute(channel ->
                     {
                        channel.send(offsetRequest.underlying());
                        final kafka.api.OffsetResponse underlyingResponse =
                           kafka.api.OffsetResponse.readFrom(channel.receive().buffer());

                        LOG.debug("Received offset response: {}", underlyingResponse);

                        return new OffsetResponse(underlyingResponse);
                     }));
}
 

private void refreshLagMetrics() {
  for (Map.Entry<SystemStreamPartition, Long> e : nextOffsets.entrySet()) {
    SystemStreamPartition ssp = e.getKey();
    Long offset = e.getValue();
    TopicAndPartition tp = new TopicAndPartition(ssp.getStream(), ssp.getPartition().getPartitionId());
    Long lag = latestLags.get(ssp);
    LOG.trace("Latest offset of {} is  {}; lag = {}", ssp, offset, lag);
    if (lag != null && offset != null && lag >= 0) {
      long streamEndOffset = offset + lag;
      // update the metrics
      kafkaConsumerMetrics.setHighWatermarkValue(tp, streamEndOffset);
      kafkaConsumerMetrics.setLagValue(tp, lag);
    }
  }
}
 

private long getOffset(KafkaPartition partition,
    Map<TopicAndPartition, PartitionOffsetRequestInfo> offsetRequestInfo)
    throws KafkaOffsetRetrievalFailureException {
  SimpleConsumer consumer = this.getSimpleConsumer(partition.getLeader().getHostAndPort());
  for (int i = 0; i < this.fetchOffsetRetries; i++) {
    try {
      OffsetResponse offsetResponse = consumer.getOffsetsBefore(new OffsetRequest(offsetRequestInfo,
          kafka.api.OffsetRequest.CurrentVersion(), this.clientName));
      if (offsetResponse.hasError()) {
        throw new RuntimeException(
            "offsetReponse has error: " + offsetResponse.errorCode(partition.getTopicName(), partition.getId()));
      }
      return offsetResponse.offsets(partition.getTopicName(), partition.getId())[0];
    } catch (Exception e) {
      LOG.warn(
          String.format("Fetching offset for partition %s has failed %d time(s). Reason: %s", partition, i + 1, e));
      if (i < this.fetchOffsetRetries - 1) {
        try {
          Thread.sleep((long) ((i + Math.random()) * 1000));
        } catch (InterruptedException e2) {
          LOG.error("Caught interrupted exception between retries of getting latest offsets. " + e2);
        }
      }
    }
  }
  throw new KafkaOffsetRetrievalFailureException(
      String.format("Fetching offset for partition %s has failed.", partition));
}
 

public long getLatestOffset(SimpleConsumer consumer, String topic, int partition, String clientName) {
    TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partition);
    Map<TopicAndPartition, kafka.api.PartitionOffsetRequestInfo> requestInfo = new HashMap<>();
    requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(kafka.api.OffsetRequest.LatestTime(), 1));
    kafka.javaapi.OffsetRequest request = new kafka.javaapi.OffsetRequest(requestInfo, kafka.api.OffsetRequest.CurrentVersion(), clientName);
    OffsetResponse response = consumer.getOffsetsBefore(request);
    if (response.hasError()) {
        throw new RuntimeException("Error fetching data offset from the broker. Reason: " + response.errorCode(topic, partition) );
    }
    long[] offsets = response.offsets(topic, partition);
    return offsets[0];
}
 

private long getOffset(boolean earliest) throws InterruptedException {
    TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partitionId);
    Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
    requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(
        earliest ? kafka.api.OffsetRequest.EarliestTime() : kafka.api.OffsetRequest.LatestTime(), 1));
    OffsetRequest request = new OffsetRequest(
            requestInfo, kafka.api.OffsetRequest.CurrentVersion(), clientId);
    OffsetResponse response = null;
    try {
        response = consumer.getOffsetsBefore(request);
    } catch (Exception e) {
        // e could be an instance of ClosedByInterruptException as SimpleConsumer.getOffsetsBefore uses nio
        if (Thread.interrupted()) {
            logger.info("catch exception of {} with interrupted in getOffset({}) for {} - {}",
                    e.getClass().getName(), earliest, topic, partitionId);

            throw new InterruptedException();
        }

        logger.error("caught exception in getOffsetsBefore {} - {}", topic, partitionId, e);
        return -1;
    }
    if (response.hasError()) {
        logger.error("error fetching data Offset from the Broker {}. reason: {}", leaderBroker.host(), response.errorCode(topic, partitionId));
        return -1;
    }
    long[] offsets = response.offsets(topic, partitionId);
    return earliest ? offsets[0] : offsets[offsets.length - 1];
}
 

@Test
public void getPartitionsManyPartitions() {
    int partitionsPerTopic = 25;
    List<TopicAndPartition> partitions = new ArrayList<>();
    for (int i = 0; i < partitionsPerTopic; ++i) {
        partitions.add(new TopicAndPartition(topic, i));
    }

    client.createTopic(topic, partitionsPerTopic, 1);
    assertThat(client.getPartitions(), hasItems(partitions.toArray(new TopicAndPartition[partitions.size()])));
}
 

private FetchRequest createFetchRequest(KafkaPartition partition, long nextOffset) {
  TopicAndPartition topicAndPartition = new TopicAndPartition(partition.getTopicName(), partition.getId());
  PartitionFetchInfo partitionFetchInfo = new PartitionFetchInfo(nextOffset, this.bufferSize);
  Map<TopicAndPartition, PartitionFetchInfo> fetchInfo =
      Collections.singletonMap(topicAndPartition, partitionFetchInfo);
  return new FetchRequest(this.fetchCorrelationId, this.clientName, this.fetchTimeoutMillis, this.fetchMinBytes,
      fetchInfo);
}