kafka.javaapi.OffsetResponse Java Examples

private long getLastOffset(SimpleConsumer consumer, String topic, int partition,
                                 long whichTime, String clientName) throws StageException {
  try {
    TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partition);
    Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<>();
    requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(whichTime, 1));
    kafka.javaapi.OffsetRequest request = new kafka.javaapi.OffsetRequest(
      requestInfo, kafka.api.OffsetRequest.CurrentVersion(), clientName);
    OffsetResponse response = consumer.getOffsetsBefore(request);

    if (response.hasError()) {
      LOG.error(KafkaErrors.KAFKA_22.getMessage(), consumer.host() + ":" + consumer.port(),
        response.errorCode(topic, partition));
      return 0;
    }
    long[] offsets = response.offsets(topic, partition);
    return offsets[0];
  } catch (Exception e) {
    LOG.error(KafkaErrors.KAFKA_30.getMessage(), e.toString(), e);
    throw new StageException(KafkaErrors.KAFKA_30, e.toString(), e);
  }
}
 

private long getEarliestOffsetFromKafka(String topic, int partition, long startTime) {
  LOG.info("getEarliestOffsetFromKafka.");
  TopicAndPartition tp = new TopicAndPartition(topic, partition);

  Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo =
      new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();

  requestInfo.put(tp, new PartitionOffsetRequestInfo(startTime, 1));

  OffsetRequest req = new OffsetRequest(
      requestInfo, kafka.api.OffsetRequest.CurrentVersion(), getClientName());

  OffsetResponse resp = consumer.getOffsetsBefore(req);

  if (resp.hasError()) {
    LOG.error("error when fetching offset: " + resp.errorCode(topic, partition)); //xxx
    return 0;
  }
  LOG.info("Earliest offset " + resp.offsets(topic, partition)[0]);
  return resp.offsets(topic, partition)[0];
}
 

private long findLastOffset(TopicPartition topicPartition, SimpleConsumer consumer) {
    TopicAndPartition topicAndPartition = new TopicAndPartition(topicPartition.getTopic(),
            topicPartition.getPartition());
    Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo =
            new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
    requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(
            kafka.api.OffsetRequest.LatestTime(), 1));
    final String clientName = getClientName(topicPartition);
    OffsetRequest request = new OffsetRequest(requestInfo,
                                              kafka.api.OffsetRequest.CurrentVersion(),
                                              clientName);
    OffsetResponse response = consumer.getOffsetsBefore(request);

    if (response.hasError()) {
        throw new RuntimeException("Error fetching offset data. Reason: " +
                response.errorCode(topicPartition.getTopic(), topicPartition.getPartition()));
    }
    long[] offsets = response.offsets(topicPartition.getTopic(),
            topicPartition.getPartition());
    return offsets[0] - 1;
}
 

public static long getLastOffset(SimpleConsumer consumer, String topic, int partition,
                                 long whichTime, String clientName) {
    TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partition);
    Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
    requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(whichTime, 1));
    kafka.javaapi.OffsetRequest request = new kafka.javaapi.OffsetRequest(
            requestInfo, kafka.api.OffsetRequest.CurrentVersion(), clientName);
    OffsetResponse response = consumer.getOffsetsBefore(request);
 
    if (response.hasError()) {
        System.out.println("Error fetching data Offset Data the Broker. Reason: " + response.errorCode(topic, partition) );
        return 0;
    }
    long[] offsets = response.offsets(topic, partition);
    return offsets[0];
}
 

@Override
public long getLastOffset(long time) {
  SimpleConsumer consumer = new SimpleConsumer(uri.getHost(), uri.getPort(), 60000,
      1024 * 1024, "hadoop-etl");
  Map<TopicAndPartition, PartitionOffsetRequestInfo> offsetInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
  offsetInfo.put(new TopicAndPartition(topic, partition), new PartitionOffsetRequestInfo(
      time, 1));
  OffsetResponse response = consumer.getOffsetsBefore(new OffsetRequest(offsetInfo,
      kafka.api.OffsetRequest.CurrentVersion(),"hadoop-etl"));
  long[] endOffset = response.offsets(topic, partition);
  consumer.close();
  if(endOffset.length == 0)
  {
    log.info("The exception is thrown because the latest offset retunred zero for topic : " + topic + " and partition " + partition);
  }
  this.latestOffset = endOffset[0];
  return endOffset[0];
}
 

@Override
public long getEarliestOffset() {
  if (this.earliestOffset == -2 && uri != null) {
    // TODO : Make the hardcoded paramters configurable
    SimpleConsumer consumer = new SimpleConsumer(uri.getHost(), uri.getPort(), 60000,
        1024 * 1024, "hadoop-etl");
    Map<TopicAndPartition, PartitionOffsetRequestInfo> offsetInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
    offsetInfo.put(new TopicAndPartition(topic, partition), new PartitionOffsetRequestInfo(
        kafka.api.OffsetRequest.EarliestTime(), 1));
    OffsetResponse response = consumer
        .getOffsetsBefore(new OffsetRequest(offsetInfo, kafka.api.OffsetRequest
            .CurrentVersion(), "hadoop-etl"));
    long[] endOffset = response.offsets(topic, partition);
    consumer.close();
    this.earliestOffset = endOffset[0];
    return endOffset[0];
  } else {
    return this.earliestOffset;
  }
}
 

private long getLastOffset(SimpleConsumer consumer, String topic, int partition,
                                 long whichTime, String clientName) throws StageException {
  try {
    TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partition);
    Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<>();
    requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(whichTime, 1));
    kafka.javaapi.OffsetRequest request = new kafka.javaapi.OffsetRequest(
      requestInfo, kafka.api.OffsetRequest.CurrentVersion(), clientName);
    OffsetResponse response = consumer.getOffsetsBefore(request);

    if (response.hasError()) {
      LOG.error(KafkaErrors.KAFKA_22.getMessage(), consumer.host() + ":" + consumer.port(),
        response.errorCode(topic, partition));
      return 0;
    }
    long[] offsets = response.offsets(topic, partition);
    return offsets[0];
  } catch (Exception e) {
    LOG.error(KafkaErrors.KAFKA_30.getMessage(), e.toString(), e);
    throw new StageException(KafkaErrors.KAFKA_30, e.toString(), e);
  }
}
 

/**
 * @param consumer
 * @param topic
 * @param partition
 * @param whichTime
 * @param clientName
 * @return 0 if consumer is null at this time
 */
public static long getLastOffset(SimpleConsumer consumer, String topic, int partition, long whichTime, String clientName)
{
  if (consumer == null) {
    return 0;
  }
  TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partition);
  Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
  requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(whichTime, 1));
  OffsetRequest request = new OffsetRequest(requestInfo, kafka.api.OffsetRequest.CurrentVersion(), clientName);
  OffsetResponse response = consumer.getOffsetsBefore(request);

  if (response.hasError()) {
    logger.error("Error fetching data Offset Data the Broker. Reason: " + response.errorCode(topic, partition));
    return 0;
  }
  long[] offsets = response.offsets(topic, partition);
  return offsets[0];
}
 

/**
 * 请求取topic偏移
 *
 * @param broker
 * @param topic
 * @param brokerPartitions
 * @return
 */
public static OffsetResponse sendOffsetRequest(Broker broker, Topic topic,
                                               List<Partition> brokerPartitions, long time) {

    PartitionOffsetRequestInfo requestInfo = new PartitionOffsetRequestInfo(time, 1);

    final OffsetRequest offsetRequest = new OffsetRequest(
            brokerPartitions.stream()
                    .collect(Collectors.toMap(
                            partition -> new TopicAndPartition(topic.getName(), partition.getId()),
                            partition -> requestInfo)), (short) 0, "kafkaMonitor");

    logger.debug("Sending offset request: {}", offsetRequest);
    if (broker != null) {
        BlockingChannel channel = getChannel(broker);
        channel.send(offsetRequest.underlying());
        final kafka.api.OffsetResponse underlyingResponse = kafka.api.OffsetResponse.readFrom(channel.receive().payload());
        channel.disconnect();
        return new OffsetResponse(underlyingResponse);
    } else {
        return null;
    }

}
 

private long getLatestOffset(SimpleConsumer consumer, TopicAndPartition topicAndPartition) {
  Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<>();
  requestInfo.put(topicAndPartition,
      new PartitionOffsetRequestInfo(kafka.api.OffsetRequest.LatestTime(), 1));
  kafka.javaapi.OffsetRequest request = new kafka.javaapi.OffsetRequest(requestInfo,
      kafka.api.OffsetRequest.CurrentVersion(), consumer.clientId());
  OffsetResponse response = consumer.getOffsetsBefore(request);

  if (response.hasError()) {
    logger.warn("Failed to fetch offset for {} due to {}", topicAndPartition,
        response.errorCode(topicAndPartition.topic(), topicAndPartition.partition()));
    return -1;
  }

  long[] offsets = response.offsets(topicAndPartition.topic(), topicAndPartition.partition());
  return offsets[0];
}
 

private static long getLastOffset(SimpleConsumer consumer, String topic, int partition,
                                  long whichTime) {
    TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partition);
    Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
    requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(whichTime, 1));
    kafka.javaapi.OffsetRequest request = new kafka.javaapi.OffsetRequest(requestInfo,
        kafka.api.OffsetRequest.CurrentVersion(), CLIENT_ID);
    OffsetResponse response = consumer.getOffsetsBefore(request);

    if (response.hasError()) {
        System.out.println("Error fetching data Offset Data the Broker. Reason: "
                           + response.errorCode(topic, partition));
        return 0;
    }
    long[] offsets = response.offsets(topic, partition);
    return offsets[0];
}
 

public static long getLastOffset(SimpleConsumer consumer, String topic, int partition, long whichTime,
      String clientName) {
	TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partition);
	Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
	requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(whichTime, 1));
	kafka.javaapi.OffsetRequest request = new kafka.javaapi.OffsetRequest(requestInfo,
	      kafka.api.OffsetRequest.CurrentVersion(), clientName);
	OffsetResponse response = consumer.getOffsetsBefore(request);

	if (response.hasError()) {
		System.out.println("Error fetching data Offset Data the Broker. Reason: "
		      + response.errorCode(topic, partition));
		return 0;
	}
	long[] offsets = response.offsets(topic, partition);
	return offsets[0];
}
 

private static long getLastOffset(SimpleConsumer consumer, String topic, int partition, long whichTime) {
	TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partition);
	Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
	requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(whichTime, 1));
	kafka.javaapi.OffsetRequest request = new kafka.javaapi.OffsetRequest(requestInfo,
			kafka.api.OffsetRequest.CurrentVersion(), CLIENT_ID);
	OffsetResponse response = consumer.getOffsetsBefore(request);

	if (response.hasError()) {
		System.out.println(
				"Error fetching data Offset Data the Broker. Reason: " + response.errorCode(topic, partition));
		return 0;
	}
	long[] offsets = response.offsets(topic, partition);
	return offsets[0];
}
 

public long getLastOffset() throws InterruptedException {
    OffsetResponse response = null;
    Broker previousLeader = leaderBroker;
    while (true) {
        TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partitionId);
        Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
        requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(kafka.api.OffsetRequest.LatestTime(), 1));
        kafka.javaapi.OffsetRequest request = new kafka.javaapi.OffsetRequest(
                requestInfo, kafka.api.OffsetRequest.CurrentVersion(), clientId);

        ensureConsumer(previousLeader);
        try {
            response = consumer.getOffsetsBefore(request);
        } catch (Exception e) {
            // e could be an instance of ClosedByInterruptException as SimpleConsumer.fetch uses nio
            if (Thread.interrupted()) {
                logger.info("catch exception of {} with interrupted in getLastOffset for {} - {}",
                        e.getClass().getName(), topic, partitionId);

                throw new InterruptedException();
            }
            logger.warn("caughte exception in getLastOffset {} - {}", topic, partitionId, e);
            response = null;
        }
        if (response == null || response.hasError()) {
            short errorCode = response != null ? response.errorCode(topic, partitionId) : ErrorMapping.UnknownCode();

            logger.warn("Error fetching data Offset for {} - {}, the Broker. Reason: {}",
                    topic, partitionId, errorCode);

            stopConsumer();
            previousLeader = leaderBroker;
            leaderBroker = null;
            continue;
        }
        break;
    }
    long[] offsets = response.offsets(topic, partitionId);
    return offsets[offsets.length - 1];
}
 

private OffsetResponse findOffsets(long timestamp, int maxSegments, List<TopicAndPartition> taps) {
   Map<TopicAndPartition, PartitionOffsetRequestInfo> request =
         new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
   for(TopicAndPartition tap: taps) {
      PartitionOffsetRequestInfo offsetRequest = new PartitionOffsetRequestInfo(timestamp, maxSegments);
      request.put(tap, offsetRequest);
   }
   return consumer.getOffsetsBefore(new OffsetRequest(request, kafka.api.OffsetRequest.CurrentVersion(), consumer.clientId()));
}
 

private long getLatestOffsetFromKafka(String topic, int partition, long startTime) {
  TopicAndPartition tp = new TopicAndPartition(topic, partition);

  Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo =
      new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();

  requestInfo.put(tp, new PartitionOffsetRequestInfo(startTime, 1));

  OffsetRequest req = new OffsetRequest(
      requestInfo, kafka.api.OffsetRequest.CurrentVersion(), getClientName());

  OffsetResponse resp = consumer.getOffsetsBefore(req);

  if (resp.hasError()) {
    LOG.error("error when fetching offset: " + resp.errorCode(topic, partition)); //xxx
    // In case you wonder what the error code really means.
    // System.out.println("OffsetOutOfRangeCode()" + ErrorMapping.OffsetOutOfRangeCode());
    // System.out.println("BrokerNotAvailableCode()" + ErrorMapping.BrokerNotAvailableCode());
    // System.out.println("InvalidFetchSizeCode()" + ErrorMapping.InvalidFetchSizeCode());
    // System.out.println("InvalidMessageCode()" + ErrorMapping.InvalidMessageCode());
    // System.out.println("LeaderNotAvailableCode()" + ErrorMapping.LeaderNotAvailableCode());
    // System.out.println("MessageSizeTooLargeCode()" + ErrorMapping.MessageSizeTooLargeCode());
    // System.out.println("NotLeaderForPartitionCode()" + ErrorMapping.NotLeaderForPartitionCode());
    // System.out.println("OffsetMetadataTooLargeCode()" + ErrorMapping.OffsetMetadataTooLargeCode());
    // System.out.println("ReplicaNotAvailableCode()" + ErrorMapping.ReplicaNotAvailableCode());
    // System.out.println("RequestTimedOutCode()" + ErrorMapping.RequestTimedOutCode());
    // System.out.println("StaleControllerEpochCode()" + ErrorMapping.StaleControllerEpochCode());
    // System.out.println("UnknownCode()" + ErrorMapping.UnknownCode());
    // System.out.println("UnknownTopicOrPartitionCode()" + ErrorMapping.UnknownTopicOrPartitionCode());

    //LOG.error("error when fetching offset: " + resp.errorcode(topic, partition));
    return 0;
  }

  return resp.offsets(topic, partition)[0];
}
 

private long getOffset(KafkaPartition partition, Map<TopicAndPartition, PartitionOffsetRequestInfo> offsetRequestInfo)
    throws KafkaOffsetRetrievalFailureException {
  SimpleConsumer consumer = this.getSimpleConsumer(partition.getLeader().getHostAndPort());
  for (int i = 0; i < this.fetchOffsetRetries; i++) {
    try {
      OffsetResponse offsetResponse =
          consumer.getOffsetsBefore(new OffsetRequest(offsetRequestInfo, kafka.api.OffsetRequest.CurrentVersion(),
              this.clientName));
      if (offsetResponse.hasError()) {
        throw new RuntimeException("offsetReponse has error: "
            + offsetResponse.errorCode(partition.getTopicName(), partition.getId()));
      }
      return offsetResponse.offsets(partition.getTopicName(), partition.getId())[0];
    } catch (Exception e) {
      log.warn(String.format("Fetching offset for partition %s has failed %d time(s). Reason: %s", partition, i + 1,
          e));
      if (i < this.fetchOffsetRetries - 1) {
        try {
          Thread.sleep((long) ((i + Math.random()) * 1000));
        } catch (InterruptedException e2) {
          log.error("Caught interrupted exception between retries of getting latest offsets. " + e2);
        }
      }
    }
  }
  throw new KafkaOffsetRetrievalFailureException(String.format("Fetching offset for partition %s has failed.",
      partition));
}
 

private long getOffset(KafkaPartition partition,
    Map<TopicAndPartition, PartitionOffsetRequestInfo> offsetRequestInfo)
    throws KafkaOffsetRetrievalFailureException {
  SimpleConsumer consumer = this.getSimpleConsumer(partition.getLeader().getHostAndPort());
  for (int i = 0; i < this.fetchOffsetRetries; i++) {
    try {
      OffsetResponse offsetResponse = consumer.getOffsetsBefore(new OffsetRequest(offsetRequestInfo,
          kafka.api.OffsetRequest.CurrentVersion(), this.clientName));
      if (offsetResponse.hasError()) {
        throw new RuntimeException(
            "offsetReponse has error: " + offsetResponse.errorCode(partition.getTopicName(), partition.getId()));
      }
      return offsetResponse.offsets(partition.getTopicName(), partition.getId())[0];
    } catch (Exception e) {
      LOG.warn(
          String.format("Fetching offset for partition %s has failed %d time(s). Reason: %s", partition, i + 1, e));
      if (i < this.fetchOffsetRetries - 1) {
        try {
          Thread.sleep((long) ((i + Math.random()) * 1000));
        } catch (InterruptedException e2) {
          LOG.error("Caught interrupted exception between retries of getting latest offsets. " + e2);
        }
      }
    }
  }
  throw new KafkaOffsetRetrievalFailureException(
      String.format("Fetching offset for partition %s has failed.", partition));
}
 

private long getOffset(boolean earliest) throws InterruptedException {
    TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partitionId);
    Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
    requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(
        earliest ? kafka.api.OffsetRequest.EarliestTime() : kafka.api.OffsetRequest.LatestTime(), 1));
    OffsetRequest request = new OffsetRequest(
            requestInfo, kafka.api.OffsetRequest.CurrentVersion(), clientId);
    OffsetResponse response = null;
    try {
        response = consumer.getOffsetsBefore(request);
    } catch (Exception e) {
        // e could be an instance of ClosedByInterruptException as SimpleConsumer.getOffsetsBefore uses nio
        if (Thread.interrupted()) {
            logger.info("catch exception of {} with interrupted in getOffset({}) for {} - {}",
                    e.getClass().getName(), earliest, topic, partitionId);

            throw new InterruptedException();
        }

        logger.error("caught exception in getOffsetsBefore {} - {}", topic, partitionId, e);
        return -1;
    }
    if (response.hasError()) {
        logger.error("error fetching data Offset from the Broker {}. reason: {}", leaderBroker.host(), response.errorCode(topic, partitionId));
        return -1;
    }
    long[] offsets = response.offsets(topic, partitionId);
    return earliest ? offsets[0] : offsets[offsets.length - 1];
}
 

public static long getLastOffset(SimpleConsumer consumer, String topic, int partition, long whichTime,
                                 String clientName) {
    TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partition);
    Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
    requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(whichTime, 1));
    OffsetRequest request = new OffsetRequest(requestInfo, kafka.api.OffsetRequest.CurrentVersion(), clientName);
    OffsetResponse response = consumer.getOffsetsBefore(request);

    if (response.hasError()) {
        log.error("Error fetching data Offset Data the Broker. Reason: " + response.errorCode(topic, partition));
        return 0;
    }
    long[] offsets = response.offsets(topic, partition);
    return offsets[0];
}
 

/**
 * Retrieves the last offset before the given timestamp for a given topic partition.
 *
 * @return The last offset before the given timestamp or {@code 0} if failed to do so.
 */
private long getLastOffset(TopicPartition topicPart, long timestamp) {
  BrokerInfo brokerInfo = brokerService.getLeader(topicPart.getTopic(), topicPart.getPartition());
  SimpleConsumer consumer = brokerInfo == null ? null : consumers.getUnchecked(brokerInfo);

  // If no broker, treat it as failure attempt.
  if (consumer == null) {
    LOG.warn("Failed to talk to any broker. Default offset to 0 for {}", topicPart);
    return 0L;
  }

  // Fire offset request
  OffsetRequest request = new OffsetRequest(ImmutableMap.of(
    new TopicAndPartition(topicPart.getTopic(), topicPart.getPartition()),
    new PartitionOffsetRequestInfo(timestamp, 1)
  ), kafka.api.OffsetRequest.CurrentVersion(), consumer.clientId());

  OffsetResponse response = consumer.getOffsetsBefore(request);

  // Retrieve offsets from response
  long[] offsets = response.hasError() ? null : response.offsets(topicPart.getTopic(), topicPart.getPartition());
  if (offsets == null || offsets.length <= 0) {
    short errorCode = response.errorCode(topicPart.getTopic(), topicPart.getPartition());

    // If the topic partition doesn't exists, use offset 0 without logging error.
    if (errorCode != ErrorMapping.UnknownTopicOrPartitionCode()) {
      consumers.refresh(brokerInfo);
      LOG.warn("Failed to fetch offset for {} with timestamp {}. Error: {}. Default offset to 0.",
               topicPart, timestamp, errorCode);
    }
    return 0L;
  }

  LOG.debug("Offset {} fetched for {} with timestamp {}.", offsets[0], topicPart, timestamp);
  return offsets[0];
}
 

public long getLatestOffset(SimpleConsumer consumer, String topic, int partition, String clientName) {
    TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partition);
    Map<TopicAndPartition, kafka.api.PartitionOffsetRequestInfo> requestInfo = new HashMap<>();
    requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(kafka.api.OffsetRequest.LatestTime(), 1));
    kafka.javaapi.OffsetRequest request = new kafka.javaapi.OffsetRequest(requestInfo, kafka.api.OffsetRequest.CurrentVersion(), clientName);
    OffsetResponse response = consumer.getOffsetsBefore(request);
    if (response.hasError()) {
        throw new RuntimeException("Error fetching data offset from the broker. Reason: " + response.errorCode(topic, partition));
    }
    long[] offsets = response.offsets(topic, partition);
    return offsets[0];
}
 

private long getLastOffset(SimpleConsumer consumer, String topic, int partition, long whichTime, String clientName) {
    TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partition);
    Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<>();
    requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(whichTime, 1));
    OffsetRequest request = new OffsetRequest(requestInfo, kafka.api.OffsetRequest.CurrentVersion(), clientName);
    OffsetResponse response = consumer.getOffsetsBefore(request);
    if (response.hasError()) {
        System.out.println("error fetching data offset data the broker. reason: " + response.errorCode(topic, partition));
        return 0;
    }
    long[] offsets = response.offsets(topic, partition);
    return offsets[0];
}
 

@Override
public OffsetInfo processPartition(String group, String topic, String partitionId) {
	OffsetInfo offsetInfo = null;
	Tuple2<String, Stat> offset_stat = readZkData(
			ZkUtils.ConsumersPath() + "/" + group + "/" + "offsets/" + topic + "/" + partitionId);
	if (offset_stat == null) {
		return null;
	}
	if (System.currentTimeMillis() - offset_stat._2().getMtime() > excludeByLastSeen) {
		// TODO 对于最后一次消费时间为一周前的，直接抛弃。是否维护一个被排除的partition 列表？
		return null;
	}
	ZkDataAndStat dataAndStat = ZKUtils
			.readDataMaybeNull(ZkUtils.ConsumersPath() + "/" + group + "/" + "owners/" + topic + "/" + partitionId);
	try {
		Integer leader = (Integer) ZKUtils.getZKUtilsFromKafka()
				.getLeaderForPartition(topic, Integer.parseInt(partitionId)).get();

		SimpleConsumer consumer = null;
		if (consumerMap.containsKey(leader)) {
			consumer = consumerMap.get(leader);
		} else {
			consumer = getConsumer(leader);
			consumerMap.put(leader, consumer);
		}

		TopicAndPartition topicAndPartition = new TopicAndPartition(topic, Integer.parseInt(partitionId));

		Map<TopicAndPartition, PartitionOffsetRequestInfo> tpMap = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
		tpMap.put(topicAndPartition, new PartitionOffsetRequestInfo(
				earliest ? kafka.api.OffsetRequest.EarliestTime() : kafka.api.OffsetRequest.LatestTime(), 1));
		OffsetRequest request = new OffsetRequest(tpMap, kafka.api.OffsetRequest.CurrentVersion(),
				String.format("%s_%s_%s", topic, partitionId, clientId));
		OffsetResponse response = null;
		response = consumer.getOffsetsBefore(request);
		if (response.hasError()) {
			LOG.error("error fetching data Offset from the Broker {}. reason: {}", leader,
					response.errorCode(topic, Integer.parseInt(partitionId)));
			throw new RuntimeException("fetching offset error!");
		}
		long[] offsets = response.offsets(topic, Integer.parseInt(partitionId));

		if (dataAndStat.getData() == null) { // Owner not available
			// TODO dataAndStat that partition may not have any owner
			offsetInfo = new OffsetInfo(group, topic, Integer.parseInt(partitionId),
					Long.parseLong(offset_stat._1()), offsets[offsets.length - 1], "NA",
					offset_stat._2().getCtime(), offset_stat._2().getMtime());
		} else {
			offsetInfo = new OffsetInfo(group, topic, Integer.parseInt(partitionId),
					Long.parseLong(offset_stat._1()), offsets[offsets.length - 1], dataAndStat.getData(),
					offset_stat._2().getCtime(), offset_stat._2().getMtime());
		}
	} catch (Exception e) {

		if (e instanceof ZkNoNodeException) {

		} else if (e instanceof NoNodeException) {
			// TODO dataAndStat that partition may not have any owner

		} else if (e instanceof BrokerNotAvailableException) {
			// TODO broker id -1 ? Alerting???
			LOG.warn(String.format("Get leader partition for [group: %s, topic: %s, partition: %s] faild!", group,
					topic, partitionId), e.getMessage());
			if (dataAndStat.getData() == null) { // Owner not available
				// TODO dataAndStat that partition may not have any owner
				offsetInfo = new OffsetInfo(group, topic, Integer.parseInt(partitionId),
						Long.parseLong(offset_stat._1()), -1l, "NA", offset_stat._2().getCtime(),
						offset_stat._2().getMtime());
			} else {
				offsetInfo = new OffsetInfo(group, topic, Integer.parseInt(partitionId),
						Long.parseLong(offset_stat._1()), -1l, dataAndStat.getData(), offset_stat._2().getCtime(),
						offset_stat._2().getMtime());
			}
		} else {
			throw new RuntimeException("Something went wrong!" + e);
		}
	}
	return offsetInfo;
}
 

/**
 * Fetches the numeric Kafka offset for this partition for a symbolic name ("largest" or "smallest").
 *
 * @param offsetCriteria
 * @param timeoutMillis Timeout in milliseconds
 * @throws java.util.concurrent.TimeoutException If the operation could not be completed within {@code timeoutMillis}
 * milliseconds
 * @return An offset
 */
@Override
public synchronized StreamPartitionMsgOffset fetchStreamPartitionOffset(@Nonnull OffsetCriteria offsetCriteria, long timeoutMillis)
    throws java.util.concurrent.TimeoutException {
  Preconditions.checkState(isPartitionProvided,
      "Cannot fetch partition offset. StreamMetadataProvider created without partition information");
  Preconditions.checkNotNull(offsetCriteria);

  final long offsetRequestTime;
  if (offsetCriteria.isLargest()) {
    offsetRequestTime = kafka.api.OffsetRequest.LatestTime();
  } else if (offsetCriteria.isSmallest()) {
    offsetRequestTime = kafka.api.OffsetRequest.EarliestTime();
  } else {
    throw new IllegalArgumentException("Unknown initial offset value " + offsetCriteria.toString());
  }

  int kafkaErrorCount = 0;
  final int MAX_KAFKA_ERROR_COUNT = 10;

  final long endTime = System.currentTimeMillis() + timeoutMillis;

  while (System.currentTimeMillis() < endTime) {
    // Try to get into a state where we're connected to Kafka
    while (_currentState.getStateValue() != KafkaConnectionHandler.ConsumerState.CONNECTED_TO_PARTITION_LEADER
        && System.currentTimeMillis() < endTime) {
      _currentState.process();
    }

    if (_currentState.getStateValue() != KafkaConnectionHandler.ConsumerState.CONNECTED_TO_PARTITION_LEADER
        && endTime <= System.currentTimeMillis()) {
      throw new TimeoutException();
    }

    // Send the offset request to Kafka
    OffsetRequest request = new OffsetRequest(Collections.singletonMap(new TopicAndPartition(_topic, _partition),
        new PartitionOffsetRequestInfo(offsetRequestTime, 1)), kafka.api.OffsetRequest.CurrentVersion(), _clientId);
    OffsetResponse offsetResponse;
    try {
      offsetResponse = _simpleConsumer.getOffsetsBefore(request);
    } catch (Exception e) {
      _currentState.handleConsumerException(e);
      continue;
    }

    final short errorCode = offsetResponse.errorCode(_topic, _partition);

    if (errorCode == Errors.NONE.code()) {
      long offset = offsetResponse.offsets(_topic, _partition)[0];
      if (offset == 0L) {
        LOGGER.warn("Fetched offset of 0 for topic {} and partition {}, is this a newly created topic?", _topic,
            _partition);
      }
      return new LongMsgOffset(offset);
    } else if (errorCode == Errors.LEADER_NOT_AVAILABLE.code()) {
      // If there is no leader, it'll take some time for a new leader to be elected, wait 100 ms before retrying
      Uninterruptibles.sleepUninterruptibly(100, TimeUnit.MILLISECONDS);
    } else {
      // Retry after a short delay
      kafkaErrorCount++;

      if (MAX_KAFKA_ERROR_COUNT < kafkaErrorCount) {
        throw exceptionForKafkaErrorCode(errorCode);
      }

      Uninterruptibles.sleepUninterruptibly(100, TimeUnit.MILLISECONDS);
    }
  }

  throw new TimeoutException();
}
 

@Override
public OffsetResponse getOffsetsBefore(OffsetRequest request) {
  throw new RuntimeException("Unimplemented!");
}