Java Code Examples for kafka.javaapi.OffsetResponse#hasError()
The following examples show how to use
kafka.javaapi.OffsetResponse#hasError() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
| Source File: KafkaLowLevelConsumer08.java From datacollector with Apache License 2.0 | 6 votes |
|
private long getLastOffset(SimpleConsumer consumer, String topic, int partition,
long whichTime, String clientName) throws StageException {
try {
TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partition);
Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<>();
requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(whichTime, 1));
kafka.javaapi.OffsetRequest request = new kafka.javaapi.OffsetRequest(
requestInfo, kafka.api.OffsetRequest.CurrentVersion(), clientName);
OffsetResponse response = consumer.getOffsetsBefore(request);
if (response.hasError()) {
LOG.error(KafkaErrors.KAFKA_22.getMessage(), consumer.host() + ":" + consumer.port(),
response.errorCode(topic, partition));
return 0;
}
long[] offsets = response.offsets(topic, partition);
return offsets[0];
} catch (Exception e) {
LOG.error(KafkaErrors.KAFKA_30.getMessage(), e.toString(), e);
throw new StageException(KafkaErrors.KAFKA_30, e.toString(), e);
}
}
Example 2
| Source File: ScribeConsumer.java From Scribengin with GNU Affero General Public License v3.0 | 6 votes |
|
private long getEarliestOffsetFromKafka(String topic, int partition, long startTime) {
LOG.info("getEarliestOffsetFromKafka.");
TopicAndPartition tp = new TopicAndPartition(topic, partition);
Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo =
new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
requestInfo.put(tp, new PartitionOffsetRequestInfo(startTime, 1));
OffsetRequest req = new OffsetRequest(
requestInfo, kafka.api.OffsetRequest.CurrentVersion(), getClientName());
OffsetResponse resp = consumer.getOffsetsBefore(req);
if (resp.hasError()) {
LOG.error("error when fetching offset: " + resp.errorCode(topic, partition)); //xxx
return 0;
}
LOG.info("Earliest offset " + resp.offsets(topic, partition)[0]);
return resp.offsets(topic, partition)[0];
}
Example 3
| Source File: LegacyKafkaClient.java From secor with Apache License 2.0 | 6 votes |
|
private long findLastOffset(TopicPartition topicPartition, SimpleConsumer consumer) {
TopicAndPartition topicAndPartition = new TopicAndPartition(topicPartition.getTopic(),
topicPartition.getPartition());
Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo =
new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(
kafka.api.OffsetRequest.LatestTime(), 1));
final String clientName = getClientName(topicPartition);
OffsetRequest request = new OffsetRequest(requestInfo,
kafka.api.OffsetRequest.CurrentVersion(),
clientName);
OffsetResponse response = consumer.getOffsetsBefore(request);
if (response.hasError()) {
throw new RuntimeException("Error fetching offset data. Reason: " +
response.errorCode(topicPartition.getTopic(), topicPartition.getPartition()));
}
long[] offsets = response.offsets(topicPartition.getTopic(),
topicPartition.getPartition());
return offsets[0] - 1;
}
Example 4
| Source File: NativeSimpleConsumer.java From spring-kafka-demo with Apache License 2.0 | 6 votes |
|
public static long getLastOffset(SimpleConsumer consumer, String topic, int partition,
long whichTime, String clientName) {
TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partition);
Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(whichTime, 1));
kafka.javaapi.OffsetRequest request = new kafka.javaapi.OffsetRequest(
requestInfo, kafka.api.OffsetRequest.CurrentVersion(), clientName);
OffsetResponse response = consumer.getOffsetsBefore(request);
if (response.hasError()) {
System.out.println("Error fetching data Offset Data the Broker. Reason: " + response.errorCode(topic, partition) );
return 0;
}
long[] offsets = response.offsets(topic, partition);
return offsets[0];
}
Example 5
| Source File: KafkaLowLevelConsumer09.java From datacollector with Apache License 2.0 | 6 votes |
|
private long getLastOffset(SimpleConsumer consumer, String topic, int partition,
long whichTime, String clientName) throws StageException {
try {
TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partition);
Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<>();
requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(whichTime, 1));
kafka.javaapi.OffsetRequest request = new kafka.javaapi.OffsetRequest(
requestInfo, kafka.api.OffsetRequest.CurrentVersion(), clientName);
OffsetResponse response = consumer.getOffsetsBefore(request);
if (response.hasError()) {
LOG.error(KafkaErrors.KAFKA_22.getMessage(), consumer.host() + ":" + consumer.port(),
response.errorCode(topic, partition));
return 0;
}
long[] offsets = response.offsets(topic, partition);
return offsets[0];
} catch (Exception e) {
LOG.error(KafkaErrors.KAFKA_30.getMessage(), e.toString(), e);
throw new StageException(KafkaErrors.KAFKA_30, e.toString(), e);
}
}
Example 6
| Source File: KafkaMetadataUtil.java From attic-apex-malhar with Apache License 2.0 | 6 votes |
|
/**
* @param consumer
* @param topic
* @param partition
* @param whichTime
* @param clientName
* @return 0 if consumer is null at this time
*/
public static long getLastOffset(SimpleConsumer consumer, String topic, int partition, long whichTime, String clientName)
{
if (consumer == null) {
return 0;
}
TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partition);
Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(whichTime, 1));
OffsetRequest request = new OffsetRequest(requestInfo, kafka.api.OffsetRequest.CurrentVersion(), clientName);
OffsetResponse response = consumer.getOffsetsBefore(request);
if (response.hasError()) {
logger.error("Error fetching data Offset Data the Broker. Reason: " + response.errorCode(topic, partition));
return 0;
}
long[] offsets = response.offsets(topic, partition);
return offsets[0];
}
Example 7
| Source File: ZkConsumerCommand.java From jeesuite-libs with Apache License 2.0 | 6 votes |
|
private static long getLastOffset(SimpleConsumer consumer, String topic, int partition, long whichTime) {
TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partition);
Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(whichTime, 1));
kafka.javaapi.OffsetRequest request = new kafka.javaapi.OffsetRequest(requestInfo,
kafka.api.OffsetRequest.CurrentVersion(), CLIENT_ID);
OffsetResponse response = consumer.getOffsetsBefore(request);
if (response.hasError()) {
System.out.println(
"Error fetching data Offset Data the Broker. Reason: " + response.errorCode(topic, partition));
return 0;
}
long[] offsets = response.offsets(topic, partition);
return offsets[0];
}
Example 8
| Source File: OffsetMonitor.java From uReplicator with Apache License 2.0 | 6 votes |
|
private long getLatestOffset(SimpleConsumer consumer, TopicAndPartition topicAndPartition) {
Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<>();
requestInfo.put(topicAndPartition,
new PartitionOffsetRequestInfo(kafka.api.OffsetRequest.LatestTime(), 1));
kafka.javaapi.OffsetRequest request = new kafka.javaapi.OffsetRequest(requestInfo,
kafka.api.OffsetRequest.CurrentVersion(), consumer.clientId());
OffsetResponse response = consumer.getOffsetsBefore(request);
if (response.hasError()) {
logger.warn("Failed to fetch offset for {} due to {}", topicAndPartition,
response.errorCode(topicAndPartition.topic(), topicAndPartition.partition()));
return -1;
}
long[] offsets = response.offsets(topicAndPartition.topic(), topicAndPartition.partition());
return offsets[0];
}
Example 9
| Source File: ZkConsumerCommand.java From azeroth with Apache License 2.0 | 6 votes |
|
private static long getLastOffset(SimpleConsumer consumer, String topic, int partition,
long whichTime) {
TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partition);
Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(whichTime, 1));
kafka.javaapi.OffsetRequest request = new kafka.javaapi.OffsetRequest(requestInfo,
kafka.api.OffsetRequest.CurrentVersion(), CLIENT_ID);
OffsetResponse response = consumer.getOffsetsBefore(request);
if (response.hasError()) {
System.out.println("Error fetching data Offset Data the Broker. Reason: "
+ response.errorCode(topic, partition));
return 0;
}
long[] offsets = response.offsets(topic, partition);
return offsets[0];
}
Example 10
| Source File: SimpleConsumerExample.java From hermes with Apache License 2.0 | 6 votes |
|
public static long getLastOffset(SimpleConsumer consumer, String topic, int partition, long whichTime,
String clientName) {
TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partition);
Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(whichTime, 1));
kafka.javaapi.OffsetRequest request = new kafka.javaapi.OffsetRequest(requestInfo,
kafka.api.OffsetRequest.CurrentVersion(), clientName);
OffsetResponse response = consumer.getOffsetsBefore(request);
if (response.hasError()) {
System.out.println("Error fetching data Offset Data the Broker. Reason: "
+ response.errorCode(topic, partition));
return 0;
}
long[] offsets = response.offsets(topic, partition);
return offsets[0];
}
Example 11
| Source File: MetricSystemTest.java From eagle with Apache License 2.0 | 5 votes |
|
private long getLastOffset(SimpleConsumer consumer, String topic, int partition, long whichTime, String clientName) {
TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partition);
Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<>();
requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(whichTime, 1));
OffsetRequest request = new OffsetRequest(requestInfo, kafka.api.OffsetRequest.CurrentVersion(), clientName);
OffsetResponse response = consumer.getOffsetsBefore(request);
if (response.hasError()) {
System.out.println("error fetching data offset data the broker. reason: " + response.errorCode(topic, partition));
return 0;
}
long[] offsets = response.offsets(topic, partition);
return offsets[0];
}
Example 12
| Source File: KafkaLatestOffsetFetcher.java From eagle with Apache License 2.0 | 5 votes |
|
public long getLatestOffset(SimpleConsumer consumer, String topic, int partition, String clientName) {
TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partition);
Map<TopicAndPartition, kafka.api.PartitionOffsetRequestInfo> requestInfo = new HashMap<>();
requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(kafka.api.OffsetRequest.LatestTime(), 1));
kafka.javaapi.OffsetRequest request = new kafka.javaapi.OffsetRequest(requestInfo, kafka.api.OffsetRequest.CurrentVersion(), clientName);
OffsetResponse response = consumer.getOffsetsBefore(request);
if (response.hasError()) {
throw new RuntimeException("Error fetching data offset from the broker. Reason: " + response.errorCode(topic, partition));
}
long[] offsets = response.offsets(topic, partition);
return offsets[0];
}
Example 13
| Source File: SimpleKafkaConsumer.java From twill with Apache License 2.0 | 5 votes |
|
/**
* Retrieves the last offset before the given timestamp for a given topic partition.
*
* @return The last offset before the given timestamp or {@code 0} if failed to do so.
*/
private long getLastOffset(TopicPartition topicPart, long timestamp) {
BrokerInfo brokerInfo = brokerService.getLeader(topicPart.getTopic(), topicPart.getPartition());
SimpleConsumer consumer = brokerInfo == null ? null : consumers.getUnchecked(brokerInfo);
// If no broker, treat it as failure attempt.
if (consumer == null) {
LOG.warn("Failed to talk to any broker. Default offset to 0 for {}", topicPart);
return 0L;
}
// Fire offset request
OffsetRequest request = new OffsetRequest(ImmutableMap.of(
new TopicAndPartition(topicPart.getTopic(), topicPart.getPartition()),
new PartitionOffsetRequestInfo(timestamp, 1)
), kafka.api.OffsetRequest.CurrentVersion(), consumer.clientId());
OffsetResponse response = consumer.getOffsetsBefore(request);
// Retrieve offsets from response
long[] offsets = response.hasError() ? null : response.offsets(topicPart.getTopic(), topicPart.getPartition());
if (offsets == null || offsets.length <= 0) {
short errorCode = response.errorCode(topicPart.getTopic(), topicPart.getPartition());
// If the topic partition doesn't exists, use offset 0 without logging error.
if (errorCode != ErrorMapping.UnknownTopicOrPartitionCode()) {
consumers.refresh(brokerInfo);
LOG.warn("Failed to fetch offset for {} with timestamp {}. Error: {}. Default offset to 0.",
topicPart, timestamp, errorCode);
}
return 0L;
}
LOG.debug("Offset {} fetched for {} with timestamp {}.", offsets[0], topicPart, timestamp);
return offsets[0];
}
Example 14
| Source File: SimpleKafkaMessageListener.java From micro-integrator with Apache License 2.0 | 5 votes |
|
public static long getLastOffset(SimpleConsumer consumer, String topic, int partition, long whichTime,
String clientName) {
TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partition);
Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(whichTime, 1));
OffsetRequest request = new OffsetRequest(requestInfo, kafka.api.OffsetRequest.CurrentVersion(), clientName);
OffsetResponse response = consumer.getOffsetsBefore(request);
if (response.hasError()) {
log.error("Error fetching data Offset Data the Broker. Reason: " + response.errorCode(topic, partition));
return 0;
}
long[] offsets = response.offsets(topic, partition);
return offsets[0];
}
Example 15
| Source File: KafkaSimpleConsumer.java From Pistachio with Apache License 2.0 | 5 votes |
|
private long getOffset(boolean earliest) throws InterruptedException {
TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partitionId);
Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(
earliest ? kafka.api.OffsetRequest.EarliestTime() : kafka.api.OffsetRequest.LatestTime(), 1));
OffsetRequest request = new OffsetRequest(
requestInfo, kafka.api.OffsetRequest.CurrentVersion(), clientId);
OffsetResponse response = null;
try {
response = consumer.getOffsetsBefore(request);
} catch (Exception e) {
// e could be an instance of ClosedByInterruptException as SimpleConsumer.getOffsetsBefore uses nio
if (Thread.interrupted()) {
logger.info("catch exception of {} with interrupted in getOffset({}) for {} - {}",
e.getClass().getName(), earliest, topic, partitionId);
throw new InterruptedException();
}
logger.error("caught exception in getOffsetsBefore {} - {}", topic, partitionId, e);
return -1;
}
if (response.hasError()) {
logger.error("error fetching data Offset from the Broker {}. reason: {}", leaderBroker.host(), response.errorCode(topic, partitionId));
return -1;
}
long[] offsets = response.offsets(topic, partitionId);
return earliest ? offsets[0] : offsets[offsets.length - 1];
}
Example 16
| Source File: KafkaSimpleConsumer.java From Pistachio with Apache License 2.0 | 5 votes |
|
public long getLastOffset() throws InterruptedException {
OffsetResponse response = null;
Broker previousLeader = leaderBroker;
while (true) {
TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partitionId);
Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(kafka.api.OffsetRequest.LatestTime(), 1));
kafka.javaapi.OffsetRequest request = new kafka.javaapi.OffsetRequest(
requestInfo, kafka.api.OffsetRequest.CurrentVersion(), clientId);
ensureConsumer(previousLeader);
try {
response = consumer.getOffsetsBefore(request);
} catch (Exception e) {
// e could be an instance of ClosedByInterruptException as SimpleConsumer.fetch uses nio
if (Thread.interrupted()) {
logger.info("catch exception of {} with interrupted in getLastOffset for {} - {}",
e.getClass().getName(), topic, partitionId);
throw new InterruptedException();
}
logger.warn("caughte exception in getLastOffset {} - {}", topic, partitionId, e);
response = null;
}
if (response == null || response.hasError()) {
short errorCode = response != null ? response.errorCode(topic, partitionId) : ErrorMapping.UnknownCode();
logger.warn("Error fetching data Offset for {} - {}, the Broker. Reason: {}",
topic, partitionId, errorCode);
stopConsumer();
previousLeader = leaderBroker;
leaderBroker = null;
continue;
}
break;
}
long[] offsets = response.offsets(topic, partitionId);
return offsets[offsets.length - 1];
}
Example 17
| Source File: KafkaWrapper.java From incubator-gobblin with Apache License 2.0 | 5 votes |
|
private long getOffset(KafkaPartition partition,
Map<TopicAndPartition, PartitionOffsetRequestInfo> offsetRequestInfo)
throws KafkaOffsetRetrievalFailureException {
SimpleConsumer consumer = this.getSimpleConsumer(partition.getLeader().getHostAndPort());
for (int i = 0; i < this.fetchOffsetRetries; i++) {
try {
OffsetResponse offsetResponse = consumer.getOffsetsBefore(new OffsetRequest(offsetRequestInfo,
kafka.api.OffsetRequest.CurrentVersion(), this.clientName));
if (offsetResponse.hasError()) {
throw new RuntimeException(
"offsetReponse has error: " + offsetResponse.errorCode(partition.getTopicName(), partition.getId()));
}
return offsetResponse.offsets(partition.getTopicName(), partition.getId())[0];
} catch (Exception e) {
LOG.warn(
String.format("Fetching offset for partition %s has failed %d time(s). Reason: %s", partition, i + 1, e));
if (i < this.fetchOffsetRetries - 1) {
try {
Thread.sleep((long) ((i + Math.random()) * 1000));
} catch (InterruptedException e2) {
LOG.error("Caught interrupted exception between retries of getting latest offsets. " + e2);
}
}
}
}
throw new KafkaOffsetRetrievalFailureException(
String.format("Fetching offset for partition %s has failed.", partition));
}
Example 18
| Source File: Kafka08ConsumerClient.java From incubator-gobblin with Apache License 2.0 | 5 votes |
|
private long getOffset(KafkaPartition partition, Map<TopicAndPartition, PartitionOffsetRequestInfo> offsetRequestInfo)
throws KafkaOffsetRetrievalFailureException {
SimpleConsumer consumer = this.getSimpleConsumer(partition.getLeader().getHostAndPort());
for (int i = 0; i < this.fetchOffsetRetries; i++) {
try {
OffsetResponse offsetResponse =
consumer.getOffsetsBefore(new OffsetRequest(offsetRequestInfo, kafka.api.OffsetRequest.CurrentVersion(),
this.clientName));
if (offsetResponse.hasError()) {
throw new RuntimeException("offsetReponse has error: "
+ offsetResponse.errorCode(partition.getTopicName(), partition.getId()));
}
return offsetResponse.offsets(partition.getTopicName(), partition.getId())[0];
} catch (Exception e) {
log.warn(String.format("Fetching offset for partition %s has failed %d time(s). Reason: %s", partition, i + 1,
e));
if (i < this.fetchOffsetRetries - 1) {
try {
Thread.sleep((long) ((i + Math.random()) * 1000));
} catch (InterruptedException e2) {
log.error("Caught interrupted exception between retries of getting latest offsets. " + e2);
}
}
}
}
throw new KafkaOffsetRetrievalFailureException(String.format("Fetching offset for partition %s has failed.",
partition));
}
Example 19
| Source File: ScribeConsumer.java From Scribengin with GNU Affero General Public License v3.0 | 5 votes |
|
private long getLatestOffsetFromKafka(String topic, int partition, long startTime) {
TopicAndPartition tp = new TopicAndPartition(topic, partition);
Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo =
new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
requestInfo.put(tp, new PartitionOffsetRequestInfo(startTime, 1));
OffsetRequest req = new OffsetRequest(
requestInfo, kafka.api.OffsetRequest.CurrentVersion(), getClientName());
OffsetResponse resp = consumer.getOffsetsBefore(req);
if (resp.hasError()) {
LOG.error("error when fetching offset: " + resp.errorCode(topic, partition)); //xxx
// In case you wonder what the error code really means.
// System.out.println("OffsetOutOfRangeCode()" + ErrorMapping.OffsetOutOfRangeCode());
// System.out.println("BrokerNotAvailableCode()" + ErrorMapping.BrokerNotAvailableCode());
// System.out.println("InvalidFetchSizeCode()" + ErrorMapping.InvalidFetchSizeCode());
// System.out.println("InvalidMessageCode()" + ErrorMapping.InvalidMessageCode());
// System.out.println("LeaderNotAvailableCode()" + ErrorMapping.LeaderNotAvailableCode());
// System.out.println("MessageSizeTooLargeCode()" + ErrorMapping.MessageSizeTooLargeCode());
// System.out.println("NotLeaderForPartitionCode()" + ErrorMapping.NotLeaderForPartitionCode());
// System.out.println("OffsetMetadataTooLargeCode()" + ErrorMapping.OffsetMetadataTooLargeCode());
// System.out.println("ReplicaNotAvailableCode()" + ErrorMapping.ReplicaNotAvailableCode());
// System.out.println("RequestTimedOutCode()" + ErrorMapping.RequestTimedOutCode());
// System.out.println("StaleControllerEpochCode()" + ErrorMapping.StaleControllerEpochCode());
// System.out.println("UnknownCode()" + ErrorMapping.UnknownCode());
// System.out.println("UnknownTopicOrPartitionCode()" + ErrorMapping.UnknownTopicOrPartitionCode());
//LOG.error("error when fetching offset: " + resp.errorcode(topic, partition));
return 0;
}
return resp.offsets(topic, partition)[0];
}
Example 20
| Source File: ZKOffsetGetter.java From kmanager with Apache License 2.0 | 4 votes |
|
@Override
public OffsetInfo processPartition(String group, String topic, String partitionId) {
OffsetInfo offsetInfo = null;
Tuple2<String, Stat> offset_stat = readZkData(
ZkUtils.ConsumersPath() + "/" + group + "/" + "offsets/" + topic + "/" + partitionId);
if (offset_stat == null) {
return null;
}
if (System.currentTimeMillis() - offset_stat._2().getMtime() > excludeByLastSeen) {
// TODO 对于最后一次消费时间为一周前的，直接抛弃。是否维护一个被排除的partition 列表？
return null;
}
ZkDataAndStat dataAndStat = ZKUtils
.readDataMaybeNull(ZkUtils.ConsumersPath() + "/" + group + "/" + "owners/" + topic + "/" + partitionId);
try {
Integer leader = (Integer) ZKUtils.getZKUtilsFromKafka()
.getLeaderForPartition(topic, Integer.parseInt(partitionId)).get();
SimpleConsumer consumer = null;
if (consumerMap.containsKey(leader)) {
consumer = consumerMap.get(leader);
} else {
consumer = getConsumer(leader);
consumerMap.put(leader, consumer);
}
TopicAndPartition topicAndPartition = new TopicAndPartition(topic, Integer.parseInt(partitionId));
Map<TopicAndPartition, PartitionOffsetRequestInfo> tpMap = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
tpMap.put(topicAndPartition, new PartitionOffsetRequestInfo(
earliest ? kafka.api.OffsetRequest.EarliestTime() : kafka.api.OffsetRequest.LatestTime(), 1));
OffsetRequest request = new OffsetRequest(tpMap, kafka.api.OffsetRequest.CurrentVersion(),
String.format("%s_%s_%s", topic, partitionId, clientId));
OffsetResponse response = null;
response = consumer.getOffsetsBefore(request);
if (response.hasError()) {
LOG.error("error fetching data Offset from the Broker {}. reason: {}", leader,
response.errorCode(topic, Integer.parseInt(partitionId)));
throw new RuntimeException("fetching offset error!");
}
long[] offsets = response.offsets(topic, Integer.parseInt(partitionId));
if (dataAndStat.getData() == null) { // Owner not available
// TODO dataAndStat that partition may not have any owner
offsetInfo = new OffsetInfo(group, topic, Integer.parseInt(partitionId),
Long.parseLong(offset_stat._1()), offsets[offsets.length - 1], "NA",
offset_stat._2().getCtime(), offset_stat._2().getMtime());
} else {
offsetInfo = new OffsetInfo(group, topic, Integer.parseInt(partitionId),
Long.parseLong(offset_stat._1()), offsets[offsets.length - 1], dataAndStat.getData(),
offset_stat._2().getCtime(), offset_stat._2().getMtime());
}
} catch (Exception e) {
if (e instanceof ZkNoNodeException) {
} else if (e instanceof NoNodeException) {
// TODO dataAndStat that partition may not have any owner
} else if (e instanceof BrokerNotAvailableException) {
// TODO broker id -1 ? Alerting???
LOG.warn(String.format("Get leader partition for [group: %s, topic: %s, partition: %s] faild!", group,
topic, partitionId), e.getMessage());
if (dataAndStat.getData() == null) { // Owner not available
// TODO dataAndStat that partition may not have any owner
offsetInfo = new OffsetInfo(group, topic, Integer.parseInt(partitionId),
Long.parseLong(offset_stat._1()), -1l, "NA", offset_stat._2().getCtime(),
offset_stat._2().getMtime());
} else {
offsetInfo = new OffsetInfo(group, topic, Integer.parseInt(partitionId),
Long.parseLong(offset_stat._1()), -1l, dataAndStat.getData(), offset_stat._2().getCtime(),
offset_stat._2().getMtime());
}
} else {
throw new RuntimeException("Something went wrong!" + e);
}
}
return offsetInfo;
}
