org.apache.kafka.clients.consumer.OffsetOutOfRangeException Java Examples

/**
 * Calling nextRecord calls fillBuffer() which has special handling for OutOfRangeExceptions.  There is some
 * recursion in this method, but it should not go on forever and yield a StackOverflow exception.  This test verifies
 * that when the KafkaConsumer is relentlessly throwing OutOfRangeExceptions that we stop trying to fill the buffer
 * after five attempts and do not have any other errors.
 */
@Test
public void testNextRecordWithRecursiveOutOfRangeException() {
    // Create mock KafkaConsumer instance
    @SuppressWarnings("unchecked")
    final KafkaConsumer<byte[], byte[]> mockKafkaConsumer = (KafkaConsumer<byte[], byte[]>) mock(KafkaConsumer.class);

    Mockito.when(mockKafkaConsumer.assignment()).thenReturn(
        Stream.of(new TopicPartition("Foobar", 0)).collect(Collectors.toSet())
    );
    Mockito.when(mockKafkaConsumer.partitionsFor(topicName)).thenReturn(Arrays.asList(
        new PartitionInfo(topicName, 0, null, null, null)
    ));
    Mockito.when(mockKafkaConsumer.poll(300)).thenThrow(
        new OffsetOutOfRangeException(new HashMap<>())
    );

    final PersistenceAdapter persistenceAdapter = new InMemoryPersistenceAdapter();
    persistenceAdapter.open(new HashMap<>());

    // Create our consumer
    final Consumer consumer = new Consumer(mockKafkaConsumer);
    consumer.open(
        getDefaultConfig(topicName),
        getDefaultVSpoutId(),
        getDefaultConsumerCohortDefinition(),
        persistenceAdapter,
        new LogRecorder(),
        null
    );

    final Record record = consumer.nextRecord();

    assertNull(record);

    Mockito.verify(mockKafkaConsumer, Mockito.times(5)).poll(300);

    consumer.close();
}
 

private long positionMain(TopicPartition partition, Duration timeout) {
  // Not handling large message here. The position will be actual position.
  while (true) { // In kafka 0.10.x we can get an unbounded number of invalid offset exception
    try {
      if (timeout == null) {
        return _kafkaConsumer.position(partition);
      } else {
        return _kafkaConsumer.position(partition, timeout);
      }
    } catch (OffsetOutOfRangeException | NoOffsetForPartitionException oe) {
      handleInvalidOffsetException(oe);
    }
  }
}
 

/**
 * Internal method used to fill internal message buffer from kafka.
 *
 * Limited by the number of trips made. This should only be called from {@link #fillBuffer()}.
 */
private void fillBuffer(final int trips) {
    // If our buffer is null, or our iterator is at the end
    if (buffer == null || !bufferIterator.hasNext()) {

        // If we have no assigned partitions to consume from, then don't call poll()
        // The underlying consumer call here does NOT make an API call, so this is safe to call within this loop.
        if (getKafkaConsumer().assignment().isEmpty()) {
            // No assigned partitions, nothing to consume :)
            return;
        }

        // Time to refill the buffer
        try {
            buffer = getKafkaConsumer().poll(300);
        } catch (OffsetOutOfRangeException outOfRangeException) {
            // Handle it
            handleOffsetOutOfRange(outOfRangeException);

            // Clear out so we can attempt next time.
            buffer = null;
            bufferIterator = null;

            // Why 5? Because it's less than 6.
            if (trips >= 5) {
                logger.error(
                    "Attempted to fill the buffer after an OffsetOutOfRangeException, but this was my fifth attempt so I'm bailing."
                );
                // nextRecord() will get called by the VirtualSpout instance soon so we're not giving up, just avoiding a StackOverflow
                // exception on this current run of checks.
                return;
            }

            fillBuffer(trips + 1);
            return;
        }

        // Create new iterator
        bufferIterator = buffer.iterator();
    }
}
 

/**
 * This method handles when a partition seek/retrieve request was out of bounds.
 * This happens in two scenarios:
 *  1 - The offset is too old and was cleaned up / removed by the broker.
 *  2 - The offset just plain does not exist.
 *
 * This is particularly nasty in that if the poll() was able to pull SOME messages from
 * SOME partitions before the exception was thrown, those messages are considered "consumed"
 * by KafkaClient, and there's no way to get them w/o seeking back to them for those partitions.
 *
 * This means when we roll back, we may replay some messages :/
 *
 * @param outOfRangeException The exception that was raised by the consumer.
 */
private void handleOffsetOutOfRange(final OffsetOutOfRangeException outOfRangeException) {
    final Set<TopicPartition> resetPartitions = new HashSet<>();

    // Loop over all the partitions in this exception
    for (final TopicPartition topicPartition : outOfRangeException.offsetOutOfRangePartitions().keySet()) {
        // The offset that was in the error
        final long exceptionOffset = outOfRangeException.offsetOutOfRangePartitions().get(topicPartition);
        // What kafka says the last offset is
        final long endingOffset = getKafkaConsumer().endOffsets(Collections.singletonList(topicPartition))
            .get(topicPartition);

        logger.warn("Offset Out of Range for partition {} at offset {}, kafka says last offset in partition is {}",
            topicPartition.partition(), exceptionOffset, endingOffset);

        // We have a hypothesis that the consumer can actually seek past the last message of the topic,
        // this yields this error and we want to catch it and try to back it up just a bit to a place that
        // we can work from.
        if (exceptionOffset >= endingOffset) {
            logger.warn(
                "OutOfRangeException yielded offset {}, which is past our ending offset of {} for {}",
                exceptionOffset,
                endingOffset,
                topicPartition
            );

            // Seek to the end we found above.  The end may have moved since we last asked, which is why we are not doing seekToEnd()
            getKafkaConsumer().seek(
                topicPartition,
                endingOffset
            );

            partitionOffsetsManager.replaceEntry(
                new ConsumerPartition(topicPartition.topic(), topicPartition.partition()),
                endingOffset
            );
        } else {
            resetPartitions.add(topicPartition);
        }
    }

    // All of the error'd partitions we need to seek to earliest available position.
    resetPartitionsToEarliest(resetPartitions);
}
 

@Test(dataProvider = "offsetResetStrategies")
public void testOffsetOutOfRangeForStrategy(LiOffsetResetStrategy strategy) throws Exception {
  createTopic(TOPIC1);
  createTopic(TOPIC2);
  produceRecordsWithKafkaProducer();
  Properties props = new Properties();
  props.setProperty("auto.offset.reset", strategy.name());
  props.setProperty("group.id", "testOffsetOutOfRange");

  TopicPartition tp = new TopicPartition(TOPIC1, 0);
  try (LiKafkaConsumer<String, String> consumer = createConsumer(props)) {
    consumer.assign(Collections.singleton(tp));
    ConsumerRecords<String, String> consumerRecords = ConsumerRecords.empty();
    consumer.seek(tp, 0);
    while (consumerRecords.isEmpty()) {
      consumerRecords = consumer.poll(1000);
    }
    consumer.seek(tp, 100000L);
    assertEquals(consumer.position(tp), 100000L);
    switch (strategy) {
      case EARLIEST:
        long expectedEarliestOffset = consumerRecords.iterator().next().offset();
        consumerRecords = ConsumerRecords.empty();
        while (consumerRecords.isEmpty()) {
          consumerRecords = consumer.poll(1000);
        }
        assertEquals(consumerRecords.iterator().next().offset(), expectedEarliestOffset,
            "The offset should have been reset to the earliest offset");
        break;
      case LATEST:
        consumer.poll(1000);
        long expectedLatestOffset = consumer.endOffsets(Collections.singleton(tp)).get(tp);
        assertEquals(consumer.position(tp), expectedLatestOffset,
            "The offset should have been reset to the latest offset");
        break;
      case LICLOSEST:
        long expectedEndOffset = consumer.endOffsets(Collections.singleton(tp)).get(tp);
        consumer.poll(1000);
        long currentPosition = consumer.position(tp);
        assertTrue(currentPosition == expectedEndOffset);
        break;
      case NONE:
        try {
          consumer.poll(1000);
          fail("OffsetOutOfRangeException should have been thrown.");
        } catch (OffsetOutOfRangeException oore) {
          // Expected
        }
        break;
      default:
        fail("Unknown reset strategy " + strategy);
        break;
    }
  }
}
 

private ConsumerRecords<K, V> poll(long timeout, boolean includeMetadataInTimeout) {
  ConsumerRecords<K, V> processedRecords;
  // We will keep polling until timeout.
  long now = System.currentTimeMillis();
  long deadline = now + timeout;
  do {
    ConsumerRecordsProcessingException crpe;

    // throw exception to user if the current active (un-paused) topic-partitions has exceptions
    Set<TopicPartition> unPausedTopicPartitions = new HashSet<>(_kafkaConsumer.assignment());
    unPausedTopicPartitions.removeAll(_kafkaConsumer.paused());
    crpe = handleRecordProcessingException(unPausedTopicPartitions);
    if (crpe != null) {
      throw crpe;
    }

    if (_autoCommitEnabled && now > _lastAutoCommitMs + _autoCommitInterval) {
      commitAsync();
      _lastAutoCommitMs = now;
    }
    ConsumerRecords<byte[], byte[]> rawRecords = ConsumerRecords.empty();
    try {
      if (includeMetadataInTimeout) {
        rawRecords = _kafkaConsumer.poll(Duration.ofMillis(deadline - now));
      } else {
        rawRecords = _kafkaConsumer.poll(deadline - now);
      }
    } catch (OffsetOutOfRangeException | NoOffsetForPartitionException oe) {
      handleInvalidOffsetException(oe);
    }

    _lastProcessedResult = _consumerRecordsProcessor.process(rawRecords);
    processedRecords = _lastProcessedResult.consumerRecords();
    // Clear the internal reference.
    _lastProcessedResult.clearRecords();
    // Rewind offset if there are processing exceptions.
    seekToCurrentOffsetsOnRecordProcessingExceptions();

    // this is an optimization
    // if no records were processed try to throw exception in current poll()
    if (processedRecords.isEmpty()) {
      crpe = handleRecordProcessingException(null);
      if (crpe != null) {
        throw crpe;
      }
    }
    now = System.currentTimeMillis();
  } while (processedRecords.isEmpty() && now < deadline);
  return processedRecords;
}
 

/**
 * This method handles the OffsetResetStrategy="LICLOSEST" offset reset strategy.
 *
 * The semantics of this strategy is defined as follows:
 * Consumer will {@link #seekToBeginning(Collection)} when InvalidOffsetException occurs due to:
 * 1. New Consumer / Expired Commit Offset
 * 2. Fall-off Start (fetch offset < LSO)
 *
 * Consumer will {@link #seekToEnd(Collection)} when InvalidOffsetException occurs due to:
 * 3a. Fall-off End (fetch offset > LEO): Consumer will seek to the end
 * 3b. Fall-off End (fetch offset <= LEO): Consumer will seek to the fetched offset
 *
 * Note: Offset to which we reset may not necessarily be a safe offset. This method invokes 2 blocking calls and does
 * ignore large-message tracking metadata. If we are unable to calculate the bounds, it will throw an
 * IllegalStateException.
 *
 * Design details can be found here - https://docs.google.com/document/d/1zKGXxZiyiRkLJ_d0FCoGALfAo0N7k3hh9NFYhJrbPsw/edit#
 * @param oe InvalidOffsetException
 */
private void handleLiClosestResetStrategy(InvalidOffsetException oe) {
  if (oe instanceof NoOffsetForPartitionException) {  // Case 1
    LOG.info("No valid offsets found. Rewinding to the earliest");
    seekToBeginning(oe.partitions());
  } else if (oe instanceof OffsetOutOfRangeException) {
    Map<TopicPartition, Long> seekBeginningPartitions = new HashMap<>();
    Map<TopicPartition, Long> seekEndPartitions = new HashMap<>();
    Map<TopicPartition, Long> seekFetchedOffsetPartitions = new HashMap<>();
    Set<TopicPartition> boundsUnknownPartitions = new HashSet<>();

    Map<TopicPartition, Long> beginningOffsets = beginningOffsets(oe.partitions());
    Map<TopicPartition, Long> endOffsets = endOffsets(oe.partitions());

    ((OffsetOutOfRangeException) oe).offsetOutOfRangePartitions().forEach((tp, fetchedOffset) -> {
      long beginningOffset = beginningOffsets.getOrDefault(tp, -1L);
      long endOffset = endOffsets.getOrDefault(tp, -1L);
      if (beginningOffset != -1L && endOffset != -1L) {
        if (beginningOffset > fetchedOffset) {  // Case 2
          seekBeginningPartitions.put(tp, beginningOffset);
          return;
        }
        if (endOffset < fetchedOffset) {  // Case 3a
          LOG.debug("Closest offset computed for topic partition {} is the log end offset {}. ", tp, fetchedOffset);
          seekEndPartitions.put(tp, endOffset);
        } else {  // Case 3b: endOffset >= fetchedOffset
          LOG.debug("Closest offset computed for topic partition {} is the fetched offset {}. ", tp, fetchedOffset);
          seekFetchedOffsetPartitions.put(tp, fetchedOffset);
        }
      } else {
        // can't handle reset if the either bound values are not known
        // ideally, this should never happen since the listoffsets protocol always returns all requested offset or none
        boundsUnknownPartitions.add(tp);
      }
    });

    if (!boundsUnknownPartitions.isEmpty()) {
      throw new IllegalStateException("Couldn't figure out the closest offset for these topic partitions " +
          boundsUnknownPartitions + "Aborting..");
    }

    if (!seekBeginningPartitions.isEmpty()) {
      LOG.info("Offsets are out of range for partitions {}. Seeking to the beginning offsets returned", seekBeginningPartitions);
      seekBeginningPartitions.forEach(this::seekAndClear);
    }
    if (!seekEndPartitions.isEmpty()) {
      LOG.info("Offsets are out of range for partitions {}. Seeking to the end offsets returned", seekEndPartitions);
      seekEndPartitions.forEach(this::seekAndClear);
    }
    if (!seekFetchedOffsetPartitions.isEmpty()) {
      LOG.info("Seeking to fetched offsets for topic partitions {}. This may indicate a potential loss of data.",
          seekFetchedOffsetPartitions.keySet());
      seekFetchedOffsetPartitions.forEach(this::seekAndClear);
    }
  } else {
    throw oe;
  }
}
 

/**
 * Handle when Kafka consumer throws OffsetOutOfRangeException. The base behavior is no-op.
 * @param e the Exception
 */
protected void handleOffsetOutOfRangeException(OffsetOutOfRangeException e) {
}