org.apache.kafka.clients.consumer.InvalidOffsetException Java Examples

private void readToEndOffsets() throws IOException {
    Set<TopicPartition> assignment = consumer.assignment();
    Map<TopicPartition, Long> endOffsets = consumer.endOffsets(assignment);
    log.trace("Reading to end of offsets {}", endOffsets);

    int count = 0;
    while (!hasReadToEndOffsets(endOffsets)) {
        try {
            count += poll();
        } catch (InvalidOffsetException e) {
            if (localCache.isPersistent()) {
                localCache.close();
                localCache.destroy();
                localCache.init();
            }
            log.warn("Seeking to beginning due to invalid offset", e);
            consumer.seekToBeginning(assignment);
            count = 0;
        }
    }
    log.info("During init or sync, processed {} records from topic {}", count, topic);
}
 

/**
 * We don't let the underlying open source consumer reset offsets so we need to do that here.
 */
private void handleInvalidOffsetException(InvalidOffsetException oe) throws InvalidOffsetException {
  switch (_offsetResetStrategy) {
    case EARLIEST:
      LOG.warn("Invalid positions of {} due to {}. Resetting position to the earliest.", oe.partitions(),
          oe.getClass().getSimpleName());
      seekToBeginning(oe.partitions());
      break;
    case LATEST:
      LOG.warn("Invalid positions of {} due to {}. Resetting position to the latest.", oe.partitions(), oe.getClass().getSimpleName());
      seekToEnd(oe.partitions());
      break;
    case LICLOSEST:
      LOG.warn("Invalid positions of {} due to {}. Resetting position to the li_closest.", oe.partitions(),
          oe.getClass().getSimpleName());
      handleLiClosestResetStrategy(oe);
      break;
    default:
      throw oe;
  }
}
 

public static ConsumerRecords<byte[], byte[]> fetchMessages(
  KafkaConfig config, KafkaConsumer<byte[], byte[]> consumer, Partition partition,
  long offset) {
  String topic = (String) config._stateConf.get(Config.KAFKA_TOPIC);
  int partitionId = partition.partition;
  TopicPartition  topicAndPartition = new TopicPartition (topic, partitionId);
  consumer.seek(topicAndPartition, offset);
  ConsumerRecords<byte[], byte[]> records;
  try {
    records = consumer.poll(config._fillFreqMs / 2);
  } catch(InvalidOffsetException ex) {
    throw new OutOfRangeException(ex.getMessage());
  } catch (Exception e) {
    if (e instanceof KafkaException || e instanceof ConnectException
      || e instanceof SocketTimeoutException || e instanceof IOException
      || e instanceof UnresolvedAddressException) {

      LOG.warn("Network error when fetching messages:", e);
      throw new FailedFetchException(e);
    } else {
      throw new RuntimeException(e);
    }
  }
  return records;
}
 

/**
 * This method handles the OffsetResetStrategy="LICLOSEST" offset reset strategy.
 *
 * The semantics of this strategy is defined as follows:
 * Consumer will {@link #seekToBeginning(Collection)} when InvalidOffsetException occurs due to:
 * 1. New Consumer / Expired Commit Offset
 * 2. Fall-off Start (fetch offset < LSO)
 *
 * Consumer will {@link #seekToEnd(Collection)} when InvalidOffsetException occurs due to:
 * 3a. Fall-off End (fetch offset > LEO): Consumer will seek to the end
 * 3b. Fall-off End (fetch offset <= LEO): Consumer will seek to the fetched offset
 *
 * Note: Offset to which we reset may not necessarily be a safe offset. This method invokes 2 blocking calls and does
 * ignore large-message tracking metadata. If we are unable to calculate the bounds, it will throw an
 * IllegalStateException.
 *
 * Design details can be found here - https://docs.google.com/document/d/1zKGXxZiyiRkLJ_d0FCoGALfAo0N7k3hh9NFYhJrbPsw/edit#
 * @param oe InvalidOffsetException
 */
private void handleLiClosestResetStrategy(InvalidOffsetException oe) {
  if (oe instanceof NoOffsetForPartitionException) {  // Case 1
    LOG.info("No valid offsets found. Rewinding to the earliest");
    seekToBeginning(oe.partitions());
  } else if (oe instanceof OffsetOutOfRangeException) {
    Map<TopicPartition, Long> seekBeginningPartitions = new HashMap<>();
    Map<TopicPartition, Long> seekEndPartitions = new HashMap<>();
    Map<TopicPartition, Long> seekFetchedOffsetPartitions = new HashMap<>();
    Set<TopicPartition> boundsUnknownPartitions = new HashSet<>();

    Map<TopicPartition, Long> beginningOffsets = beginningOffsets(oe.partitions());
    Map<TopicPartition, Long> endOffsets = endOffsets(oe.partitions());

    ((OffsetOutOfRangeException) oe).offsetOutOfRangePartitions().forEach((tp, fetchedOffset) -> {
      long beginningOffset = beginningOffsets.getOrDefault(tp, -1L);
      long endOffset = endOffsets.getOrDefault(tp, -1L);
      if (beginningOffset != -1L && endOffset != -1L) {
        if (beginningOffset > fetchedOffset) {  // Case 2
          seekBeginningPartitions.put(tp, beginningOffset);
          return;
        }
        if (endOffset < fetchedOffset) {  // Case 3a
          LOG.debug("Closest offset computed for topic partition {} is the log end offset {}. ", tp, fetchedOffset);
          seekEndPartitions.put(tp, endOffset);
        } else {  // Case 3b: endOffset >= fetchedOffset
          LOG.debug("Closest offset computed for topic partition {} is the fetched offset {}. ", tp, fetchedOffset);
          seekFetchedOffsetPartitions.put(tp, fetchedOffset);
        }
      } else {
        // can't handle reset if the either bound values are not known
        // ideally, this should never happen since the listoffsets protocol always returns all requested offset or none
        boundsUnknownPartitions.add(tp);
      }
    });

    if (!boundsUnknownPartitions.isEmpty()) {
      throw new IllegalStateException("Couldn't figure out the closest offset for these topic partitions " +
          boundsUnknownPartitions + "Aborting..");
    }

    if (!seekBeginningPartitions.isEmpty()) {
      LOG.info("Offsets are out of range for partitions {}. Seeking to the beginning offsets returned", seekBeginningPartitions);
      seekBeginningPartitions.forEach(this::seekAndClear);
    }
    if (!seekEndPartitions.isEmpty()) {
      LOG.info("Offsets are out of range for partitions {}. Seeking to the end offsets returned", seekEndPartitions);
      seekEndPartitions.forEach(this::seekAndClear);
    }
    if (!seekFetchedOffsetPartitions.isEmpty()) {
      LOG.info("Seeking to fetched offsets for topic partitions {}. This may indicate a potential loss of data.",
          seekFetchedOffsetPartitions.keySet());
      seekFetchedOffsetPartitions.forEach(this::seekAndClear);
    }
  } else {
    throw oe;
  }
}