Java Code Examples for org.apache.kafka.clients.consumer.Consumer#seekToEnd()

/**
 * createConsumer - create a new consumer
 *
 * @return
 * @throws Exception
 */
private Consumer<String, String> createConsumer() throws Exception {

    // Seek to end automatically
    TopicPartition dataTopicPartition = new TopicPartition(topicName, 0);
    List<TopicPartition> topics = Arrays.asList(dataTopicPartition);

    Properties props = ConfUtils.getProps(CONSUMER_PROPS);
    Consumer<String, String> consumer = new KafkaConsumer<>(props);
    consumer.assign(topics);

    if (offset == -1) {
        consumer.seekToEnd(topics);
        logger.info("Consumer seek to end");
    } else {
        consumer.seek(dataTopicPartition, offset);
        logger.info(String.format("read changed as offset: %s", consumer.position(dataTopicPartition)));
    }
    return consumer;
}
 

private void seekToStartPosition(Consumer<?, ?> consumer, Set<TopicPartition> partitions, String strategy) {
  // We would like to consume from latest when there is no offset. However, if we rewind due to exception, we want
  // to rewind to earliest to make sure the messages which are added after we start consuming don't get skipped
  if (_startOffsets.isPresent()) {
    _logger.info("Datastream is configured with StartPosition. Trying to start from {}", _startOffsets.get());
    seekToOffset(consumer, partitions, _startOffsets.get());
  } else {
    if (CONSUMER_AUTO_OFFSET_RESET_CONFIG_LATEST.equals(strategy)) {
      _logger.info("Datastream was not configured with StartPosition. Seeking to end for partitions: {}", partitions);
      consumer.seekToEnd(partitions);
    } else {
      _logger.info("Datastream was not configured with StartPosition. Seeking to beginning for partitions: {}",
          partitions);
      consumer.seekToBeginning(partitions);
    }
  }
}
 

@Override
public void accept(Consumer<?, ?> consumer, TopicPartition topicPartition) {
    Long absOffset = null;
    switch (seek()) {
        case FROM_BEGINNING:
            consumer.seekToBeginning(Collections.singletonList(topicPartition));
            log.info("seekToBeginning: {}", topicPartition);
            if (offset != 0L) {
                absOffset = Math.max(0L, consumer.position(topicPartition) + offset);
            }
            break;
        case FROM_END:
            consumer.seekToEnd(Collections.singletonList(topicPartition));
            log.info("seekToEnd: {}", topicPartition);
            if (offset != 0L) {
                absOffset = Math.max(0L, consumer.position(topicPartition) + offset);
            }
            break;
        case FROM_CURRENT:
            if (offset != 0L) {
                absOffset = Math.max(0L, consumer.position(topicPartition) + offset);
            }
            break;
        case TO_ABSOLUTE:
            absOffset = offset;
            break;
    }
    if (absOffset != null) {
        log.info("seek: {} to offset: {}", topicPartition, absOffset);
        consumer.seek(topicPartition, absOffset);
    }
}
 

/**
 * 取得尾部offset（LEO）
 *
 * @param topic
 * @param partition
 * @return
 */
private long findLogEndOffset(String topic, int partition) {
    Consumer consumer = getConsumer();
    TopicPartition topicPartition = new TopicPartition(topic, partition);
    List tpList = new ArrayList();
    tpList.add(topicPartition);
    consumer.assign(tpList);
    consumer.seekToEnd(tpList);
    Long longEndOffset = consumer.position(topicPartition);
    return longEndOffset;
}
 

public static Consumer<String, byte[]> createConsumer(Properties props, String subscribeTopic) throws Exception {
    TopicPartition topicPartition = new TopicPartition(subscribeTopic, 0);
    List<TopicPartition> topicPartitions = Arrays.asList(topicPartition);
    Consumer<String, byte[]> consumer = new KafkaConsumer<>(props);
    // consumer.subscribe(Arrays.asList(subscribeTopics.split(",")));
    consumer.assign(topicPartitions);
    // consumer都是在topo启动时创建。当Topo重启，目前的策略是对于kafka中未处理的msg放弃。不再消费。所以seek to end。
    consumer.seekToEnd(topicPartitions);
    return consumer;
}
 

public static Consumer<String, byte[]> createConsumer(Properties props, String subscribeTopic) {
    List<TopicPartition> topicPartitions = new ArrayList<>();
    for (String topic : subscribeTopic.split(",")) {
        topicPartitions.add(new TopicPartition(topic, 0));
    }
    Consumer<String, byte[]> consumer = new KafkaConsumer<>(props);
    consumer.assign(topicPartitions);
    consumer.seekToEnd(topicPartitions);
    return consumer;
}