org.apache.kafka.clients.consumer.OffsetCommitCallback Java Examples

private void commitOffsets(Map<TopicPartition, OffsetAndMetadata> offsets,
                           boolean ignoreConsumerHighWatermark,
                           OffsetCommitCallback callback,
                           boolean sync,
                           Duration timeout) {
  Map<TopicPartition, OffsetAndMetadata> offsetsToCommit =
      _consumerRecordsProcessor.safeOffsetsToCommit(offsets, ignoreConsumerHighWatermark);
  if (sync) {
    if (timeout == null) {
      LOG.trace("Committing offsets synchronously: {}", offsetsToCommit);
      _kafkaConsumer.commitSync(offsetsToCommit);
    } else {
      LOG.trace("Committing offsets synchronously with timeout {} ms: {}", timeout.toMillis(), offsetsToCommit);
      _kafkaConsumer.commitSync(offsetsToCommit, timeout);
    }
  } else {
    LOG.trace("Committing offsets asynchronously: {}", offsetsToCommit);
    _offsetCommitCallback.setUserCallback(callback);
    _kafkaConsumer.commitAsync(offsetsToCommit, _offsetCommitCallback);
  }
}
 

@Override
public void commitAsync(Map<TopicPartition, OffsetAndMetadata> offsets, OffsetCommitCallback callback) {
  try (
      @SuppressWarnings("unused") CloseableLock uLock = new CloseableLock(userLock);
      @SuppressWarnings("unused") CloseableLock srLock = new CloseableLock(delegateLock.readLock())
  ) {
    verifyOpen();
    delegate.commitAsync(offsets, callback);
  }
}
 

private void commitOffsets(ConsumerWorker worker) {
	
	KafkaConsumer<String, Serializable> consumer = worker.consumer;
	if(worker.isCommiting())return;
	worker.setCommiting(true);
	try {

		if(worker.uncommittedOffsetMap.isEmpty())return ;
		
		logger.debug("committing the offsets : {}", worker.uncommittedOffsetMap);
		consumer.commitAsync(worker.uncommittedOffsetMap, new OffsetCommitCallback() {
			@Override
			public void onComplete(Map<TopicPartition, OffsetAndMetadata> offsets, Exception exception) {
				//
				worker.setCommiting(false);
				if(exception == null){
					worker.resetUncommittedOffsetMap();
					logger.debug("committed the offsets : {}",offsets);
				}else{
					logger.error("committ the offsets error",exception);
				}
			}
		});
	} finally {
		
	}
}
 

@Override
public void commitAsync(OffsetCommitCallback callback) {
  try (
      @SuppressWarnings("unused") CloseableLock uLock = new CloseableLock(userLock);
      @SuppressWarnings("unused") CloseableLock srLock = new CloseableLock(delegateLock.readLock())
  ) {
    verifyOpen();
    delegate.commitAsync(callback);
  }
}
 

private OffsetAndMetadata commitAndRetrieveOffsets(
    LiKafkaConsumer<String, String> consumer,
    TopicPartition tp, Map<TopicPartition,
    OffsetAndMetadata> offsetMap) throws Exception {
  final AtomicBoolean callbackFired = new AtomicBoolean(false);
  final AtomicReference<Exception> offsetCommitIssue = new AtomicReference<>(null);
  OffsetAndMetadata committed = null;
  long now = System.currentTimeMillis();
  long deadline = now + TimeUnit.MINUTES.toMillis(1);
  while (System.currentTimeMillis() < deadline) {
    //call commitAsync, wait for a NON-NULL return value (see https://issues.apache.org/jira/browse/KAFKA-6183)
    OffsetCommitCallback commitCallback = new OffsetCommitCallback() {
      @Override
      public void onComplete(Map<TopicPartition, OffsetAndMetadata> topicPartitionOffsetAndMetadataMap, Exception e) {
        if (e != null) {
          offsetCommitIssue.set(e);
        }
        callbackFired.set(true);
      }
    };
    if (offsetMap != null) {
      consumer.commitAsync(offsetMap, commitCallback);
    } else {
      consumer.commitAsync(commitCallback);
    }
    while (!callbackFired.get()) {
      consumer.poll(20);
    }
    Assert.assertNull(offsetCommitIssue.get(), "offset commit failed");
    committed = consumer.committed(tp);
    if (committed != null) {
      break;
    }
    Thread.sleep(100);
  }
  assertNotNull(committed, "unable to retrieve committed offsets within timeout");
  return committed;
}
 

/**
 * Use the supplied function to asynchronously consume messages from the cluster.
 *
 * @param groupId the name of the group; may not be null
 * @param clientId the name of the client; may not be null
 * @param autoOffsetReset how to pick a starting offset when there is no initial offset in ZooKeeper or if an offset is
 *        out of range; may be null for the default to be used
 * @param keyDeserializer the deserializer for the keys; may not be null
 * @param valueDeserializer the deserializer for the values; may not be null
 * @param continuation the function that determines if the consumer should continue; may not be null
 * @param offsetCommitCallback the callback that should be used after committing offsets; may be null if offsets are
 *        not to be committed
 * @param completion the function to call when the consumer terminates; may be null
 * @param topics the set of topics to consume; may not be null or empty
 * @param consumerFunction the function to consume the messages; may not be null
 */
public <K, V> void consume(String groupId, String clientId, OffsetResetStrategy autoOffsetReset,
        Deserializer<K> keyDeserializer, Deserializer<V> valueDeserializer,
        BooleanSupplier continuation, OffsetCommitCallback offsetCommitCallback, Runnable completion,
        Collection<String> topics,
        java.util.function.Consumer<ConsumerRecord<K, V>> consumerFunction) {
    Properties props = getConsumerProperties(groupId, clientId, autoOffsetReset);
    Thread t = new Thread(() -> {
        LOGGER.infof("Starting consumer %s to read messages", clientId);
        try (KafkaConsumer<K, V> consumer = new KafkaConsumer<>(props, keyDeserializer, valueDeserializer)) {
            consumer.subscribe(new ArrayList<>(topics));
            while (continuation.getAsBoolean()) {
                consumer.poll(Duration.ofMillis(10)).forEach(record -> {
                    LOGGER.infof("Consumer %s: consuming message %s", clientId, record);
                    consumerFunction.accept(record);
                    if (offsetCommitCallback != null) {
                        consumer.commitAsync(offsetCommitCallback);
                    }
                });
            }
        } finally {
            if (completion != null) {
                completion.run();
            }
            LOGGER.debugf("Stopping consumer %s", clientId);
        }
    });
    t.setName(clientId + "-thread");
    t.start();
}
 

@Override
public void commitAsync(Map<TopicPartition, OffsetAndMetadata> offsets, OffsetCommitCallback callback) {
    ObjectExtension.requireNonNull(offsets, "offsets");

    runWithoutConcurrency(() -> {
        Map<String, Map<TopicPartition, OffsetAndMetadata>> byTopic = new HashMap<>();
        offsets.forEach((tp, oam) -> byTopic.computeIfAbsent(tp.topic(), k -> new HashMap<>()).put(tp, oam));
        byTopic.forEach((t, os) -> _consumerHolders.get(t).getConsumer().commitAsync(os, callback));
    });
}
 

@Override
public void commitAsync(Map<TopicPartition, OffsetAndMetadata> map, OffsetCommitCallback offsetCommitCallback) {
}
 

@Override
public void commitAsync(OffsetCommitCallback offsetCommitCallback) {
}
 

@Override public void commitAsync(OffsetCommitCallback callback) {
  delegate.commitAsync(callback);
}
 

@Override
public void commitAsync(OffsetCommitCallback callback) {
    runWithoutConcurrency(() -> _consumerHolders.values().forEach(c -> c.getConsumer().commitAsync(callback)));
}
 

@Override public void commitAsync(Map<TopicPartition, OffsetAndMetadata> offsets,
  OffsetCommitCallback callback) {
  delegate.commitAsync(offsets, callback);
}
 

/**
 * Commit the specified offsets for the specified list of topics and partitions to Kafka.
 * @param offsets given offsets
 * @param callback Callback to invoke when the commit completes
 */
@Override
public void commitAsync(Map<TopicPartition, OffsetAndMetadata> offsets, OffsetCommitCallback callback)
{
  consumer.commitAsync(offsets, callback);
}
 

/**
 * Commit the specified offsets for the specified list of topics and partitions to Kafka.
 * @param offsets given offsets
 * @param callback Callback to invoke when the commit completes
 */
@Override
public void commitAsync(Map<TopicPartition, OffsetAndMetadata> offsets, OffsetCommitCallback callback)
{
  consumer.commitAsync(offsets, callback);
}
 

/**
 * Sample ConsumeService instance for unit testing
 * @return Sample ConsumeService object.
 * @throws Exception should the ConsumeService creation fail or throws an error / exception
 */
private ConsumeService consumeService() throws Exception {
  LOG.info("Creating an instance of Consume Service for testing..");

  ConsumerFactory consumerFactory = Mockito.mock(ConsumerFactory.class);
  AdminClient adminClient = Mockito.mock(AdminClient.class);
  KMBaseConsumer kmBaseConsumer = Mockito.mock(KMBaseConsumer.class);

  Mockito.when(consumerFactory.adminClient()).thenReturn(adminClient);
  Mockito.when(consumerFactory.latencySlaMs()).thenReturn(20000);
  Mockito.when(consumerFactory.baseConsumer()).thenReturn(kmBaseConsumer);
  Mockito.when(consumerFactory.topic()).thenReturn(TOPIC);

  /* LATENCY_PERCENTILE_MAX_MS_CONFIG, */
  Mockito.when(consumerFactory.latencyPercentileMaxMs()).thenReturn(5000);

  /* LATENCY_PERCENTILE_GRANULARITY_MS_CONFIG */
  Mockito.when(consumerFactory.latencyPercentileGranularityMs()).thenReturn(1);

  /* define return value */
  Mockito.when(kmBaseConsumer.lastCommitted()).thenReturn(MOCK_LAST_COMMITTED_OFFSET);
  Mockito.when(kmBaseConsumer.committed(Mockito.any())).thenReturn(new OffsetAndMetadata(FIRST_OFFSET));
  Mockito.doAnswer(new Answer<Void>() {
    @Override
    public Void answer(InvocationOnMock invocationOnMock) {
      OffsetCommitCallback callback = invocationOnMock.getArgument(0);
      Map<TopicPartition, OffsetAndMetadata> committedOffsets = new HashMap<>();
      committedOffsets.put(new TopicPartition(TOPIC, PARTITION), new OffsetAndMetadata(FIRST_OFFSET));
      callback.onComplete(committedOffsets, null);

      return null;
    }
  }).when(kmBaseConsumer).commitAsync(Mockito.any(OffsetCommitCallback.class));


  /* avro record to KmBaseConsumer record */
  Mockito.when(kmBaseConsumer.receive()).thenReturn(
      new BaseConsumerRecord(TOPIC, PARTITION, SECOND_OFFSET, "key",
          Utils.jsonFromFields(TOPIC, 2, 6000, "producerId", 2)));

  CompletableFuture<Void> topicPartitionResult = new CompletableFuture<>();
  topicPartitionResult.complete(null);

  return new ConsumeService(TAG_NAME_VALUE, topicPartitionResult, consumerFactory);
}
 

private void consume() throws Exception {
  /* Delay 1 second to reduce the chance that consumer creates topic before TopicManagementService */
  Thread.sleep(1000);

  Map<Integer, Long> nextIndexes = new HashMap<>();

  while (_running.get()) {
    BaseConsumerRecord record;
    try {
      record = _baseConsumer.receive();
    } catch (Exception e) {
      _sensors._consumeError.record();
      LOG.warn(_name + "/ConsumeService failed to receive record", e);
      /* Avoid busy while loop */
      //noinspection BusyWait
      Thread.sleep(CONSUME_THREAD_SLEEP_MS);
      continue;
    }

    if (record == null) continue;

    GenericRecord avroRecord = null;
    try {
      avroRecord = Utils.genericRecordFromJson(record.value());
    } catch (Exception exception) {
      LOG.error("An exception occurred while getting avro record.", exception);
    }

    if (avroRecord == null) {
      _sensors._consumeError.record();
      continue;
    }
    int partition = record.partition();
    /* Commit availability and commit latency service */
    /* Call commitAsync, wait for a NON-NULL return value (see https://issues.apache.org/jira/browse/KAFKA-6183) */
    OffsetCommitCallback commitCallback = new OffsetCommitCallback() {
      @Override
      public void onComplete(Map<TopicPartition, OffsetAndMetadata> topicPartitionOffsetAndMetadataMap, Exception kafkaException) {
        if (kafkaException != null) {
          LOG.error("Exception while trying to perform an asynchronous commit.", kafkaException);
          _commitAvailabilityMetrics._failedCommitOffsets.record();
        } else {
          _commitAvailabilityMetrics._offsetsCommitted.record();
          _commitLatencyMetrics.recordCommitComplete();
        }
      }
    };

    /* Current timestamp to perform subtraction*/
    long currTimeMillis = System.currentTimeMillis();

    /* 4 seconds consumer offset commit interval. */
    long timeDiffMillis = TimeUnit.SECONDS.toMillis(COMMIT_TIME_INTERVAL);

    if (currTimeMillis - _baseConsumer.lastCommitted() >= timeDiffMillis) {
      /* commit the consumer offset asynchronously with a callback. */
      _baseConsumer.commitAsync(commitCallback);
      _commitLatencyMetrics.recordCommitStart();
      /* Record the current time for the committed consumer offset */
      _baseConsumer.updateLastCommit();
    }
    /* Finished consumer offset commit service. */

    long index = (Long) avroRecord.get(DefaultTopicSchema.INDEX_FIELD.name());
    long currMs = System.currentTimeMillis();
    long prevMs = (Long) avroRecord.get(DefaultTopicSchema.TIME_FIELD.name());

    _sensors._recordsConsumed.record();
    _sensors._bytesConsumed.record(record.value().length());
    _sensors._recordsDelay.record(currMs - prevMs);

    if (currMs - prevMs > _latencySlaMs)
      _sensors._recordsDelayed.record();

    if (index == -1L || !nextIndexes.containsKey(partition)) {
      nextIndexes.put(partition, -1L);
      continue;
    }

    long nextIndex = nextIndexes.get(partition);

    if (nextIndex == -1 || index == nextIndex) {
      nextIndexes.put(partition, index + 1);

    } else if (index < nextIndex) {
      _sensors._recordsDuplicated.record();
    } else if (index > nextIndex) {
      nextIndexes.put(partition, index + 1);
      long numLostRecords = index - nextIndex;
      _sensors._recordsLost.record(numLostRecords);
      LOG.info("_recordsLost recorded: Avro record current index: {} at timestamp {}. Next index: {}. Lost {} records.", index, currMs, nextIndex, numLostRecords);
    }
  }
  /* end of consume() while loop */
}
 

@Override
public void commitAsync(OffsetCommitCallback callback) {
  _consumer.commitAsync(callback);
}
 

@Override
public void commitAsync(final Map<TopicPartition, OffsetAndMetadata> offsets, OffsetCommitCallback callback) {
  _consumer.commitAsync(offsets, callback);
}
 

@Override
public void commitAsync(Map<TopicPartition, OffsetAndMetadata> offsets, OffsetCommitCallback callback) {
  _delegate.commitAsync(offsets, callback);
}
 

@Override
public void commitAsync(OffsetCommitCallback callback) {
  _delegate.commitAsync(callback);
}
 

@Override
public void commitAsync(Map<TopicPartition, OffsetAndMetadata> offsets, OffsetCommitCallback callback) {
  // Ignore the high watermark.
  commitOffsets(offsets, true, callback, false, null);
}
 

@Override
public void commitAsync(OffsetCommitCallback callback) {
  // preserve the high watermark.
  commitOffsets(currentOffsetAndMetadataMap(), false, callback, false, null);
}
 

public void setUserCallback(OffsetCommitCallback userCallback) {
  _userCallback = userCallback;
}
 

@Override
public void commitAsync(Map<TopicPartition, OffsetAndMetadata> offsets,
    OffsetCommitCallback callback) {
  consumer.commitAsync(offsets, callback);
}
 

@Override
public void commitAsync(OffsetCommitCallback callback) {
  consumer.commitAsync(callback);
}
 

@Override
public void commitAsync(Map<TopicPartition, OffsetAndMetadata> offsets, OffsetCommitCallback callback) {
    kafkaConsumer.commitAsync(offsets, callback);
}
 

@Override
public void commitAsync(OffsetCommitCallback callback) {
    kafkaConsumer.commitAsync(callback);
}
 

/**
 * Commit the specified offsets for the specified list of topics and partitions to Kafka.
 * @param offsets given offsets
 * @param callback Callback to invoke when the commit completes
 */
void commitAsync(Map<TopicPartition, OffsetAndMetadata> offsets, OffsetCommitCallback callback);
 

/**
 * {@inheritDoc}
 * <p>
 * This method is large message transparent.
 * <p>
 * With large message support, the consumer will internally translate the user provided offsets to the safe
 * offsets for the corresponding partitions (see {@link #safeOffset(TopicPartition, long)}. The actual offset
 * committed to Kafka can be retrieved through method {@link #committedSafeOffset(TopicPartition)}. The arguments
 * passed to the callback is large message transparent.
 */
@Override
@InterfaceOrigin.ApacheKafka
void commitAsync(Map<TopicPartition, OffsetAndMetadata> offsets, OffsetCommitCallback callback);
 

void commitAsync(final Map<TopicPartition, OffsetAndMetadata> offsets, OffsetCommitCallback callback);