Java Code Examples for org.apache.kafka.clients.consumer.ConsumerRecords#empty()

/**
 * Polls the next element from the Handover, possibly blocking until the next element is
 * available. This method behaves similar to polling from a blocking queue.
 *
 * <p>If an exception was handed in by the producer ({@link #reportError(Throwable)}), then
 * that exception is thrown rather than an element being returned.
 *
 * @return The next element (buffer of records, never null).
 *
 * @throws ClosedException Thrown if the Handover was {@link #close() closed}.
 * @throws Exception Rethrows exceptions from the {@link #reportError(Throwable)} method.
 */
@Nonnull
public ConsumerRecords<byte[], byte[]> pollNext() throws Exception {
	synchronized (lock) {
		while (next == null && error == null) {
			lock.wait();
		}

		ConsumerRecords<byte[], byte[]> n = next;
		if (n != null) {
			next = null;
			lock.notifyAll();
			return n;
		}
		else {
			ExceptionUtils.rethrowException(error, error.getMessage());

			// this statement cannot be reached since the above method always throws an exception
			// this is only here to silence the compiler and any warnings
			return ConsumerRecords.empty();
		}
	}
}
 

/**
 * Polls the next element from the Handover, possibly blocking until the next element is
 * available. This method behaves similar to polling from a blocking queue.
 *
 * <p>If an exception was handed in by the producer ({@link #reportError(Throwable)}), then
 * that exception is thrown rather than an element being returned.
 *
 * @return The next element (buffer of records, never null).
 *
 * @throws ClosedException Thrown if the Handover was {@link #close() closed}.
 * @throws Exception Rethrows exceptions from the {@link #reportError(Throwable)} method.
 */
@Nonnull
public ConsumerRecords<byte[], byte[]> pollNext() throws Exception {
	synchronized (lock) {
		while (next == null && error == null) {
			lock.wait();
		}

		ConsumerRecords<byte[], byte[]> n = next;
		if (n != null) {
			next = null;
			lock.notifyAll();
			return n;
		}
		else {
			ExceptionUtils.rethrowException(error, error.getMessage());

			// this statement cannot be reached since the above method always throws an exception
			// this is only here to silence the compiler and any warnings
			return ConsumerRecords.empty();
		}
	}
}
 

/**
 * Polls the next element from the Handover, possibly blocking until the next element is
 * available. This method behaves similar to polling from a blocking queue.
 *
 * <p>If an exception was handed in by the producer ({@link #reportError(Throwable)}), then
 * that exception is thrown rather than an element being returned.
 *
 * @return The next element (buffer of records, never null).
 *
 * @throws ClosedException Thrown if the Handover was {@link #close() closed}.
 * @throws Exception Rethrows exceptions from the {@link #reportError(Throwable)} method.
 */
@Nonnull
public ConsumerRecords<byte[], byte[]> pollNext() throws Exception {
	synchronized (lock) {
		while (next == null && error == null) {
			lock.wait();
		}

		ConsumerRecords<byte[], byte[]> n = next;
		if (n != null) {
			next = null;
			lock.notifyAll();
			return n;
		}
		else {
			ExceptionUtils.rethrowException(error, error.getMessage());

			// this statement cannot be reached since the above method always throws an exception
			// this is only here to silence the compiler and any warnings
			return ConsumerRecords.empty();
		}
	}
}
 

/**
 * Polls the next element from the Handover, possibly blocking until the next element is
 * available. This method behaves similar to polling from a blocking queue.
 *
 * <p>If an exception was handed in by the producer ({@link #reportError(Throwable)}), then
 * that exception is thrown rather than an element being returned.
 *
 * @return The next element (buffer of records, never null).
 *
 * @throws ClosedException Thrown if the Handover was {@link #close() closed}.
 * @throws Exception Rethrows exceptions from the {@link #reportError(Throwable)} method.
 */
@Nonnull
public ConsumerRecords<byte[], byte[]> pollNext() throws Exception {
	synchronized (lock) {
		while (next == null && error == null) {
			lock.wait();
		}

		ConsumerRecords<byte[], byte[]> n = next;
		if (n != null) {
			next = null;
			lock.notifyAll();
			return n;
		}
		else {
			ExceptionUtils.rethrowException(error, error.getMessage());

			// this statement cannot be reached since the above method always throws an exception
			// this is only here to silence the compiler and any warnings
			return ConsumerRecords.empty();
		}
	}
}
 

@Test
public void testNullValue() throws Exception {
  String topic = "testNullValue";
  createTopic(topic);
  try (LiKafkaProducer<String, String> producer = createProducer(null)) {
    producer.send(new ProducerRecord<>(topic, "key", null));
  }
  Properties consumerProps = new Properties();
  consumerProps.setProperty(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");

  try (LiKafkaConsumer<String, String> consumer = createConsumer(consumerProps)) {
    consumer.subscribe(Collections.singleton(topic));
    long startMs = System.currentTimeMillis();
    ConsumerRecords<String, String> records = ConsumerRecords.empty();
    while (records.isEmpty() && System.currentTimeMillis() < startMs + 60000) {
      records = consumer.poll(Duration.ofMillis(100));
    }
    assertEquals(1, records.count());
    ConsumerRecord<String, String> record = records.iterator().next();
    assertEquals("key", record.key());
    assertNull(record.value());
  }
}
 

private void testExceptionProcessingByFunction(String topic, LiKafkaConsumer<byte[], byte[]> consumer,
    BiConsumer<LiKafkaConsumer<byte[], byte[]>, TopicPartition> testFunction) throws Exception {
  try {
    consumer.subscribe(Collections.singleton(topic));
    ConsumerRecords<byte[], byte[]> records = ConsumerRecords.empty();
    while (records.isEmpty()) {
      records = consumer.poll(Duration.ofMillis(10));
    }
    assertEquals(records.count(), 4, "Only the first message should be returned");
    assertEquals(records.iterator().next().offset(), 2L, "The offset of the first message should be 2.");
    assertEquals(consumer.position(new TopicPartition(topic, 0)), 7L, "The position should be 7");

    testFunction.accept(consumer, new TopicPartition(topic, 0));
  } finally {
    consumer.close();
  }
}
 

/**
 * Polls the next element from the Handover, possibly blocking until the next element is
 * available. This method behaves similar to polling from a blocking queue.
 *
 * <p>If an exception was handed in by the producer ({@link #reportError(Throwable)}), then
 * that exception is thrown rather than an element being returned.
 *
 * @return The next element (buffer of records, never null).
 *
 * @throws ClosedException Thrown if the Handover was {@link #close() closed}.
 * @throws Exception Rethrows exceptions from the {@link #reportError(Throwable)} method.
 */
@Nonnull
public ConsumerRecords<byte[], byte[]> pollNext() throws Exception {
	synchronized (lock) {
		while (next == null && error == null) {
			lock.wait();
		}

		ConsumerRecords<byte[], byte[]> n = next;
		if (n != null) {
			next = null;
			lock.notifyAll();
			return n;
		}
		else {
			ExceptionUtils.rethrowException(error, error.getMessage());

			// this statement cannot be reached since the above method always throws an exception
			// this is only here to silence the compiler and any warnings
			return ConsumerRecords.empty();
		}
	}
}
 

/**
 * Polls the next element from the Handover, possibly blocking until the next element is
 * available. This method behaves similar to polling from a blocking queue.
 *
 * <p>If an exception was handed in by the producer ({@link #reportError(Throwable)}), then
 * that exception is thrown rather than an element being returned.
 *
 * @return The next element (buffer of records, never null).
 *
 * @throws ClosedException Thrown if the Handover was {@link #close() closed}.
 * @throws Exception Rethrows exceptions from the {@link #reportError(Throwable)} method.
 */
@Nonnull
public ConsumerRecords<byte[], byte[]> pollNext() throws Exception {
	synchronized (lock) {
		while (next == null && error == null) {
			lock.wait();
		}

		ConsumerRecords<byte[], byte[]> n = next;
		if (n != null) {
			next = null;
			lock.notifyAll();
			return n;
		}
		else {
			ExceptionUtils.rethrowException(error, error.getMessage());

			// this statement cannot be reached since the above method always throws an exception
			// this is only here to silence the compiler and any warnings
			return ConsumerRecords.empty();
		}
	}
}
 

private ConsumerRecords<K, V> pollRecords(long timeout) {
    try {

        long currentTime = System.currentTimeMillis();

        // Exclude the first poll from the latency timer, since it takes approximately max.poll.interval.ms as the
        // consumer group is stabilizing.
        if (lastPollTime != -1L && pollCount > 1L) {
            long pollLatency = currentTime - lastPollTime;

            POLL_LATENCY.update(pollLatency);

            if (pollLatency > config.getMaxPollInterval() && LOGGER.isWarnEnabled()) {
                LOGGER.warn("{}ms has elapsed since last #poll(). This is greater than max.poll.interval.ms {}. If this " +
                        "continues you may need to increase max.poll.interval.ms", pollLatency, config.getMaxPollInterval());
            }
        }

        lastPollTime = currentTime;
        ++pollCount;

        return consumer.poll(Duration.ofMillis(timeout));
    } catch (IllegalStateException e) {
        // The Kafka consumer will throw this exception if the consumer is not currently subscribed to any topics. Return an
        // empty record collection after verifying that is in fact the case, otherwise rethrow the exception.
        if (consumer.subscription().isEmpty()) {
            LOGGER.debug("Consumer with no subscriptions polled for records.");
            return ConsumerRecords.empty();
        } else {
            throw e;
        }
    }
}
 

@Test
public void testZeroLengthValue() throws Exception {
  String topic = "testZeroLengthValue";
  createTopic(topic);

  Properties producerPropertyOverrides = new Properties();
  producerPropertyOverrides.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class.getName());

  //this is actually Producer<String, byte[]>
  try (LiKafkaProducer producer = createProducer(producerPropertyOverrides)) {
    //noinspection unchecked
    producer.send(new ProducerRecord<>(topic, "key", new byte[0])).get();
  }
  Properties consumerProps = new Properties();
  consumerProps.setProperty(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
  consumerProps.setProperty(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class.getName());

  try (LiKafkaConsumer consumer = createConsumer(consumerProps)) {
    consumer.subscribe(Collections.singleton(topic));
    long startMs = System.currentTimeMillis();
    ConsumerRecords records = ConsumerRecords.empty();
    while (records.isEmpty() && System.currentTimeMillis() < startMs + 30000) {
      records = consumer.poll(100);
    }
    assertEquals(1, records.count());
    ConsumerRecord record = (ConsumerRecord) records.iterator().next();
    assertEquals("key", record.key());
    assertEquals(((byte[]) record.value()).length, 0);
  }
}
 

@Test
public void testBootstrapWithLiClosest() throws Exception {
  createTopic(TOPIC1);
  createTopic(TOPIC2);
  produceRecordsWithKafkaProducer();
  Properties props = new Properties();
  props.setProperty("auto.offset.reset", LiOffsetResetStrategy.LICLOSEST.name());
  props.setProperty("group.id", "testBootstrapLICLOSEST");

  LiKafkaConsumer<String, String> consumer = createConsumer(props);
  TopicPartition tp = new TopicPartition(TOPIC1, 0);
  try {
    consumer.assign(Collections.singleton(tp));
    ConsumerRecords<String, String> consumerRecords = ConsumerRecords.empty();
    while (consumerRecords.isEmpty()) {
      consumerRecords = consumer.poll(1000);
    }
    long bootstrappedReset = consumerRecords.iterator().next().offset();
    consumer.seekToBeginning(Collections.singleton(tp));
    consumerRecords = ConsumerRecords.empty();
    while (consumerRecords.isEmpty()) {
      consumerRecords = consumer.poll(1000);
    }
    assertEquals(bootstrappedReset, consumerRecords.iterator().next().offset()); // because we seek to beginning above
  } finally {
    consumer.close();
  }
}
 

private static ConsumerRecords<byte[], byte[]> createTestRecords() {
	return ConsumerRecords.empty();
}
 

@Test(dataProvider = "offsetResetStrategies")
public void testOffsetOutOfRangeForStrategy(LiOffsetResetStrategy strategy) throws Exception {
  createTopic(TOPIC1);
  createTopic(TOPIC2);
  produceRecordsWithKafkaProducer();
  Properties props = new Properties();
  props.setProperty("auto.offset.reset", strategy.name());
  props.setProperty("group.id", "testOffsetOutOfRange");

  TopicPartition tp = new TopicPartition(TOPIC1, 0);
  try (LiKafkaConsumer<String, String> consumer = createConsumer(props)) {
    consumer.assign(Collections.singleton(tp));
    ConsumerRecords<String, String> consumerRecords = ConsumerRecords.empty();
    consumer.seek(tp, 0);
    while (consumerRecords.isEmpty()) {
      consumerRecords = consumer.poll(1000);
    }
    consumer.seek(tp, 100000L);
    assertEquals(consumer.position(tp), 100000L);
    switch (strategy) {
      case EARLIEST:
        long expectedEarliestOffset = consumerRecords.iterator().next().offset();
        consumerRecords = ConsumerRecords.empty();
        while (consumerRecords.isEmpty()) {
          consumerRecords = consumer.poll(1000);
        }
        assertEquals(consumerRecords.iterator().next().offset(), expectedEarliestOffset,
            "The offset should have been reset to the earliest offset");
        break;
      case LATEST:
        consumer.poll(1000);
        long expectedLatestOffset = consumer.endOffsets(Collections.singleton(tp)).get(tp);
        assertEquals(consumer.position(tp), expectedLatestOffset,
            "The offset should have been reset to the latest offset");
        break;
      case LICLOSEST:
        long expectedEndOffset = consumer.endOffsets(Collections.singleton(tp)).get(tp);
        consumer.poll(1000);
        long currentPosition = consumer.position(tp);
        assertTrue(currentPosition == expectedEndOffset);
        break;
      case NONE:
        try {
          consumer.poll(1000);
          fail("OffsetOutOfRangeException should have been thrown.");
        } catch (OffsetOutOfRangeException oore) {
          // Expected
        }
        break;
      default:
        fail("Unknown reset strategy " + strategy);
        break;
    }
  }
}
 

private ConsumerRecords<K, V> poll(long timeout, boolean includeMetadataInTimeout) {
  ConsumerRecords<K, V> processedRecords;
  // We will keep polling until timeout.
  long now = System.currentTimeMillis();
  long deadline = now + timeout;
  do {
    ConsumerRecordsProcessingException crpe;

    // throw exception to user if the current active (un-paused) topic-partitions has exceptions
    Set<TopicPartition> unPausedTopicPartitions = new HashSet<>(_kafkaConsumer.assignment());
    unPausedTopicPartitions.removeAll(_kafkaConsumer.paused());
    crpe = handleRecordProcessingException(unPausedTopicPartitions);
    if (crpe != null) {
      throw crpe;
    }

    if (_autoCommitEnabled && now > _lastAutoCommitMs + _autoCommitInterval) {
      commitAsync();
      _lastAutoCommitMs = now;
    }
    ConsumerRecords<byte[], byte[]> rawRecords = ConsumerRecords.empty();
    try {
      if (includeMetadataInTimeout) {
        rawRecords = _kafkaConsumer.poll(Duration.ofMillis(deadline - now));
      } else {
        rawRecords = _kafkaConsumer.poll(deadline - now);
      }
    } catch (OffsetOutOfRangeException | NoOffsetForPartitionException oe) {
      handleInvalidOffsetException(oe);
    }

    _lastProcessedResult = _consumerRecordsProcessor.process(rawRecords);
    processedRecords = _lastProcessedResult.consumerRecords();
    // Clear the internal reference.
    _lastProcessedResult.clearRecords();
    // Rewind offset if there are processing exceptions.
    seekToCurrentOffsetsOnRecordProcessingExceptions();

    // this is an optimization
    // if no records were processed try to throw exception in current poll()
    if (processedRecords.isEmpty()) {
      crpe = handleRecordProcessingException(null);
      if (crpe != null) {
        throw crpe;
      }
    }
    now = System.currentTimeMillis();
  } while (processedRecords.isEmpty() && now < deadline);
  return processedRecords;
}
 

private static ConsumerRecords<byte[], byte[]> createTestRecords() {
	return ConsumerRecords.empty();
}