Java Code Examples for org.apache.kafka.clients.consumer.ConsumerRecords#records()

public static void main(String[] args) {
    KafkaConsumer<String, String> consumer = new ConsumerFactory<String, String>().create();

    try {
        while (true) {
            ConsumerRecords<String, String> records = consumer.poll(Duration.ofMillis(1000));
            for (TopicPartition partition : records.partitions()) {
                List<ConsumerRecord<String, String>> partitionRecords = records.records(partition);
                for (ConsumerRecord<String, String> record : partitionRecords) {
                    //do some logical processing.
                }
                long lastConsumedOffset = partitionRecords.get(partitionRecords.size() - 1).offset();
                consumer.commitSync(Collections.singletonMap(partition,
                        new OffsetAndMetadata(lastConsumedOffset + 1)));
            }
        }
    } finally {
        consumer.close();
    }
}
 

@Override
public ConsumerRecords<String, String> onConsume(
        ConsumerRecords<String, String> records) {
    System.out.println("before:" + records);
    long now = System.currentTimeMillis();
    Map<TopicPartition, List<ConsumerRecord<String, String>>> newRecords
            = new HashMap<>();
    for (TopicPartition tp : records.partitions()) {
        List<ConsumerRecord<String, String>> tpRecords = records.records(tp);
        List<ConsumerRecord<String, String>> newTpRecords = new ArrayList<>();
        for (ConsumerRecord<String, String> record : tpRecords) {
            if (now - record.timestamp() < EXPIRE_INTERVAL) {
                newTpRecords.add(record);
            }
        }
        if (!newTpRecords.isEmpty()) {
            newRecords.put(tp, newTpRecords);
        }
    }
    return new ConsumerRecords<>(newRecords);
}
 

@Override
public void handle(Exception thrownException, ConsumerRecords<?, ?> data) {
    thrownException.printStackTrace();
    log.error(handleLogMessage(thrownException));
    if (thrownException.getCause() instanceof MethodArgumentNotValidException) {
        return;
    }
    Set<TopicPartition> topicPartitions = data.partitions();
    log.error("send failed message to dead letter");
    for (TopicPartition topicPartition : topicPartitions) {
        List<? extends ConsumerRecord<?, ?>> list = data.records(topicPartition);
        for (ConsumerRecord<?, ?> consumerRecord : list) {
            deadLetterPublishingRecoverer.accept(consumerRecord, thrownException);
        }
    }
    log.error("send failed message to dead letter successful");
}
 

/**
 * Fetches the next available {@link org.apache.kafka.clients.consumer.Consumer} from the pool and
 * issues a poll request to the broker with the specified timeout.
 *
 * @param pollTimeout information of timeout
 */
private void consumeFromTopic(long pollTimeout) {
  int consumerIndex = nextConsumerIndex.getAndUpdate((value) -> ++value % kafkaConsumers.size());
  int newMessages = 0;
  ConsumerRecords<String, ByteBuffer> polled = null;
  try {
    polled = kafkaConsumers.get(consumerIndex).getWithConsumer(c -> c.poll(pollTimeout));
  } catch (KafkaException e) {
    // TODO: Handle this which might include creating a new consumer
    logger.atWarning().withCause(e).log("Unexpected KafkaException during poll");
  }

  if (polled != null) {
    for (ConsumerRecord<String, ByteBuffer> record : polled.records(kafkaTopicName)) {
      buffer.add(record);
      newMessages++;
      queueSizeBytes.addAndGet(record.serializedValueSize());
    }
    logger.atFine().log(
        "poll(%d) from consumer %d returned %d messages (buffer=%d, bufferBytes=%d)",
        pollTimeout, consumerIndex, newMessages, buffer.size(), queueSizeBytes.get());
  }
}
 

@Override
public ConsumerRecords<String, String> onConsume(
        ConsumerRecords<String, String> records) {
    long now = System.currentTimeMillis();
    Map<TopicPartition, List<ConsumerRecord<String, String>>> newRecords
            = new HashMap<>();
    for (TopicPartition tp : records.partitions()) {
        List<ConsumerRecord<String, String>> tpRecords = records.records(tp);
        List<ConsumerRecord<String, String>> newTpRecords = new ArrayList<>();
        for (ConsumerRecord<String, String> record : tpRecords) {
            Headers headers = record.headers();
            long ttl = -1;
            for (Header header : headers) {//判断headers中是否有key为"ttl"的Header
                if (header.key().equalsIgnoreCase("ttl")) {
                    ttl = BytesUtils.bytesToLong(header.value());
                }
            }
            //消息超时判定
            if (ttl > 0 && now - record.timestamp() < ttl * 1000) {
                newTpRecords.add(record);
            } else if (ttl < 0) {//没有设置ttl,无需超时判定
                newTpRecords.add(record);
            }
        }
        if (!newTpRecords.isEmpty()) {
            newRecords.put(tp, newTpRecords);
        }
    }
    return new ConsumerRecords<>(newRecords);
}
 

@Override
public void runFetchLoop() throws Exception {
	try {
		// kick off the actual Kafka consumer
		consumerThread.start();

		while (running) {
			// this blocks until we get the next records
			// it automatically re-throws exceptions encountered in the consumer thread
			final ConsumerRecords<byte[], byte[]> records = handover.pollNext();

			// get the records for each topic partition
			for (KafkaTopicPartitionState<T, TopicPartition> partition : subscribedPartitionStates()) {

				List<ConsumerRecord<byte[], byte[]>> partitionRecords =
					records.records(partition.getKafkaPartitionHandle());

				partitionConsumerRecordsHandler(partitionRecords, partition);
			}
		}
	}
	finally {
		// this signals the consumer thread that no more work is to be done
		consumerThread.shutdown();
	}

	// on a clean exit, wait for the runner thread
	try {
		consumerThread.join();
	}
	catch (InterruptedException e) {
		// may be the result of a wake-up interruption after an exception.
		// we ignore this here and only restore the interruption state
		Thread.currentThread().interrupt();
	}
}
 

/**
 * Consume messages from a given partition of a Kafka topic, using given ReaderCallback
 */
public static void readTopic(String topic, Integer partition, String brokerList, ReaderCallback callback)
    throws Exception {
  Validate.notNull(topic);
  Validate.notNull(partition);
  Validate.notNull(brokerList);
  Validate.notNull(callback);

  KafkaConsumer<byte[], byte[]> consumer = createConsumer(brokerList);
  if (partition >= 0) {
    List<TopicPartition> topicPartitions = Collections.singletonList(new TopicPartition(topic, partition));
    consumer.assign(topicPartitions);
    consumer.seekToBeginning(topicPartitions);
  } else {
    consumer.subscribe(Collections.singletonList(topic));
  }

  boolean keepGoing = true;
  long now = System.currentTimeMillis();
  do {
    ConsumerRecords<byte[], byte[]> records = consumer.poll(1000);
    for (ConsumerRecord<byte[], byte[]> record : records.records(topic)) {
      if (!callback.onMessage(record.key(), record.value())) {
        keepGoing = false;
        break;
      }
    }

    // Guard against buggy test which can hang forever
    if (System.currentTimeMillis() - now >= DEFAULT_TIMEOUT_MS) {
      throw new TimeoutException("Timed out before reading all messages");
    }
  } while (keepGoing);
}
 

/**
 * Translate the Kafka consumer records if necessary and send the batch of records to destination.
 * @param records the Kafka consumer records
 * @param readTime the instant the records were successfully polled from the Kafka source
 */
protected void translateAndSendBatch(ConsumerRecords<?, ?> records, Instant readTime) {
  // iterate through each topic partition one at a time, for better isolation
  for (TopicPartition topicPartition : records.partitions()) {
    for (ConsumerRecord<?, ?> record : records.records(topicPartition)) {
      try {
        boolean partitionPaused;
        boolean sendFailure;
        synchronized (_sendFailureTopicPartitionExceptionMap) {
          partitionPaused = _autoPausedSourcePartitions.containsKey(topicPartition);
          sendFailure = _sendFailureTopicPartitionExceptionMap.containsKey(topicPartition);
        }
        if (partitionPaused || sendFailure) {
          _logger.warn("Abort sending for {}, auto-paused: {}, send failure: {}, rewind offset", topicPartition,
              partitionPaused, sendFailure);
          seekToLastCheckpoint(Collections.singleton(topicPartition));
          break;
        } else {
          DatastreamProducerRecord datastreamProducerRecord = translate(record, readTime);
          int numBytes = record.serializedKeySize() + record.serializedValueSize();
          sendDatastreamProducerRecord(datastreamProducerRecord, topicPartition, numBytes, null);
        }
      } catch (Exception e) {
        _logger.warn(String.format("Got exception while sending record %s, exception: ", record), e);
        rewindAndPausePartitionOnException(topicPartition, e);
        // skip other messages for this partition, but can continue processing other partitions
        break;
      }
    }
  }
}
 

@Override
public ConsumerRecords<String, String> onConsume(
        ConsumerRecords<String, String> records) {
    long now = System.currentTimeMillis();
    Map<TopicPartition, List<ConsumerRecord<String, String>>> newRecords
            = new HashMap<>();
    for (TopicPartition tp : records.partitions()) {
        List<ConsumerRecord<String, String>> tpRecords = records.records(tp);
        List<ConsumerRecord<String, String>> newTpRecords = new ArrayList<>();
        for (ConsumerRecord<String, String> record : tpRecords) {
            Headers headers = record.headers();
            long ttl = -1;
            for (Header header : headers) {//判断headers中是否有key为"ttl"的Header
                if (header.key().equalsIgnoreCase("ttl")) {
                    ttl = BytesUtils.bytesToLong(header.value());
                }
            }
            //消息超时判定
            if (ttl > 0 && now - record.timestamp() < ttl * 1000) {
                newTpRecords.add(record);
            } else {//没有设置ttl,无需超时判定
                newTpRecords.add(record);
            }
        }
        if (!newTpRecords.isEmpty()) {
            newRecords.put(tp, newTpRecords);
        }
    }
    return new ConsumerRecords<>(newRecords);
}
 

public List<MessageVO> getMessages(
        String topicName,
        int partitionId,
        long offset,
        long count,
        MessageDeserializer deserializer)
{
   try (Consumer<byte[], byte[]> consumer = createConsumer())
   {
      TopicPartition topicPartition = new TopicPartition(topicName, partitionId);
      consumer.assign(Collections.singletonList(topicPartition));
      consumer.seek(topicPartition, offset);

      // Need to adjust how many records we read based on how many are actually available in the partition.
      long maxOffset = consumer.endOffsets(Collections.singletonList(topicPartition)).get(topicPartition);

      int numRecordsToRead = (int) Math.min(count, maxOffset - offset);

      List<MessageVO> recordList = new ArrayList<>(numRecordsToRead);
      while (recordList.size() < numRecordsToRead)
      {
         ConsumerRecords<byte[], byte[]> records = consumer.poll(Duration.ofMillis(1000L));
         List<ConsumerRecord<byte[], byte[]>> returnedRecords = records.records(topicPartition);

         returnedRecords.subList(0, Math.min((int) count - recordList.size(), returnedRecords.size()))
            .stream()
            .map(record -> createMessage(record, deserializer))
            .forEach(recordList::add);
      }

      return recordList;
   }
}
 

@Override
public void runFetchLoop() throws Exception {
	try {
		final Handover handover = this.handover;

		// kick off the actual Kafka consumer
		consumerThread.start();

		while (running) {
			// this blocks until we get the next records
			// it automatically re-throws exceptions encountered in the consumer thread
			final ConsumerRecords<byte[], byte[]> records = handover.pollNext();

			// get the records for each topic partition
			for (KafkaTopicPartitionState<TopicPartition> partition : subscribedPartitionStates()) {

				List<ConsumerRecord<byte[], byte[]>> partitionRecords =
					records.records(partition.getKafkaPartitionHandle());

				for (ConsumerRecord<byte[], byte[]> record : partitionRecords) {
					final T value = deserializer.deserialize(record);

					if (deserializer.isEndOfStream(value)) {
						// end of stream signaled
						running = false;
						break;
					}

					// emit the actual record. this also updates offset state atomically
					// and deals with timestamps and watermark generation
					emitRecord(value, partition, record.offset(), record);
				}
			}
		}
	}
	finally {
		// this signals the consumer thread that no more work is to be done
		consumerThread.shutdown();
	}

	// on a clean exit, wait for the runner thread
	try {
		consumerThread.join();
	}
	catch (InterruptedException e) {
		// may be the result of a wake-up interruption after an exception.
		// we ignore this here and only restore the interruption state
		Thread.currentThread().interrupt();
	}
}
 

@Override
public void runFetchLoop() throws Exception {
	try {
		final Handover handover = this.handover;

		// kick off the actual Kafka consumer
		consumerThread.start();

		while (running) {
			// this blocks until we get the next records
			// it automatically re-throws exceptions encountered in the consumer thread
			final ConsumerRecords<byte[], byte[]> records = handover.pollNext();

			// get the records for each topic partition
			for (KafkaTopicPartitionState<T, TopicPartition> partition : subscribedPartitionStates()) {

				List<ConsumerRecord<byte[], byte[]>> partitionRecords =
					records.records(partition.getKafkaPartitionHandle());

				for (ConsumerRecord<byte[], byte[]> record : partitionRecords) {
					deserializer.deserialize(record, kafkaCollector);

					// emit the actual records. this also updates offset state atomically and emits
					// watermarks
					emitRecordsWithTimestamps(
						kafkaCollector.getRecords(),
						partition,
						record.offset(),
						record.timestamp());

					if (kafkaCollector.isEndOfStreamSignalled()) {
						// end of stream signaled
						running = false;
						break;
					}
				}
			}
		}
	}
	finally {
		// this signals the consumer thread that no more work is to be done
		consumerThread.shutdown();
	}

	// on a clean exit, wait for the runner thread
	try {
		consumerThread.join();
	}
	catch (InterruptedException e) {
		// may be the result of a wake-up interruption after an exception.
		// we ignore this here and only restore the interruption state
		Thread.currentThread().interrupt();
	}
}
 

@Override
public void afterPropertiesSet() throws Exception {

    Properties props = new Properties();
    props.put("bootstrap.servers", servers);
    props.put("group.id", group);
    props.put("client.id",client);
    props.put("enable.auto.commit", "false");
    props.put("auto.commit.interval.ms", "1000");
    props.put("session.timeout.ms", "30000");
    props.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
    props.put("value.deserializer", "io.anyway.galaxy.message.serialization.TransactionMessageDeserializer");

    consumer = new KafkaConsumer<String, TransactionMessage>(props);
    //定义事务处理的topic
    consumer.subscribe(Arrays.asList("galaxy-tx-message"));

    if(logger.isInfoEnabled()){
        logger.info("crete kafka consumer: "+consumer+" ,subscribe topic: galaxy-tx-message");
    }

    final Thread thread= new Thread(){
        @Override
        public void run() {
            for (; running; ) {
                try {
                    ConsumerRecords<String, TransactionMessage> records = consumer.poll(timeout);
                    for (TopicPartition partition : records.partitions()) {
                        List<ConsumerRecord<String, TransactionMessage>> partitionRecords = records.records(partition);
                        for (ConsumerRecord<String, TransactionMessage> each : partitionRecords) {
                            if(logger.isInfoEnabled()){
                                logger.info("kafka receive message: "+"{topic:"+each.topic()+",partition:"+partition.partition()+",offset:"+each.offset()+",value:"+each.value()+"}");
                            }
                            if (transactionMessageService.isValidMessage(each.value())) {
                                transactionMessageService.asyncHandleMessage(each.value());
                            }
                        }
                        //同步设置offset
                        long lastOffset = partitionRecords.get(partitionRecords.size() - 1).offset();
                        Map<TopicPartition, OffsetAndMetadata> offsets = Collections.singletonMap(partition, new OffsetAndMetadata(lastOffset + 1));
                        consumer.commitSync(offsets);
                        if (logger.isInfoEnabled()) {
                            logger.info("application group: " + group + " has committed offset: " + offsets);
                        }
                    }
                } catch (Throwable e) {
                    logger.error("Consumer message failed ", e);
                    try {
                        Thread.sleep(5000);
                    } catch (InterruptedException e1) {
                        // e1.printStackTrace();
                    }
                }
            }
        }
    };
    thread.setDaemon(true);
    thread.start();
}
 

@Override
public void afterPropertiesSet() throws Exception {

    Properties props = new Properties();
    props.put("bootstrap.servers", servers);
    props.put("group.id", group);
    props.put("client.id",client);
    props.put("enable.auto.commit", "false");
    props.put("auto.commit.interval.ms", "1000");
    props.put("session.timeout.ms", "30000");
    props.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
    props.put("value.deserializer", "io.anyway.galaxy.message.serialization.TransactionMessageDeserializer");

    consumer = new KafkaConsumer<String, TransactionMessage>(props);
    //定义事务处理的topic
    consumer.subscribe(Arrays.asList("galaxy-tx-message"));

    if(logger.isInfoEnabled()){
        logger.info("crete kafka consumer: "+consumer+" ,subscribe topic: galaxy-tx-message");
    }

    final Thread thread= new Thread(){
        @Override
        public void run() {
            for (; running; ) {
                try {
                    ConsumerRecords<String, TransactionMessage> records = consumer.poll(timeout);
                    for (TopicPartition partition : records.partitions()) {
                        List<ConsumerRecord<String, TransactionMessage>> partitionRecords = records.records(partition);
                        for (ConsumerRecord<String, TransactionMessage> each : partitionRecords) {
                            if(logger.isInfoEnabled()){
                                logger.info("kafka receive message: "+"{topic:"+each.topic()+",partition:"+partition.partition()+",offset:"+each.offset()+",value:"+each.value()+"}");
                            }
                            if (transactionMessageService.isValidMessage(each.value())) {
                                transactionMessageService.asyncHandleMessage(each.value());
                            }
                        }
                        //同步设置offset
                        long lastOffset = partitionRecords.get(partitionRecords.size() - 1).offset();
                        Map<TopicPartition, OffsetAndMetadata> offsets = Collections.singletonMap(partition, new OffsetAndMetadata(lastOffset + 1));
                        consumer.commitSync(offsets);
                        if (logger.isInfoEnabled()) {
                            logger.info("application group: " + group + " has committed offset: " + offsets);
                        }
                    }
                } catch (Throwable e) {
                    logger.error("Consumer message failed ", e);
                    try {
                        Thread.sleep(5000);
                    } catch (InterruptedException e1) {
                        // e1.printStackTrace();
                    }
                }
            }
        }
    };
    thread.setDaemon(true);
    thread.start();
}
 

@Test
public void testStartJob() throws Exception {
  try (MiniKafkaCluster kafkaCluster = new MiniKafkaCluster.Builder().newServer("0").build();
       MiniAthenaXCluster cluster = new MiniAthenaXCluster(StartJobITest.class.getSimpleName())) {
    kafkaCluster.start();
    setUpKafka(kafkaCluster);
    cluster.start();
    AthenaXConfiguration conf = generateConf(cluster);
    ServerContext.INSTANCE.initialize(conf);
    ServerContext.INSTANCE.start();
    try (WebServer server = new WebServer(URI.create("http://localhost:0"))) {
      server.start();
      Configuration.getDefaultApiClient().setBasePath(String.format("http://localhost:%d%s", server.port(), WebServer.BASE_PATH));
      LOG.info("AthenaX server listening on http://localhost:{}", server.port());

      JobsApi api = new JobsApi();
      String uuid = api.allocateNewJob().getJobUuid();
      JobDefinitionDesiredstate state = new JobDefinitionDesiredstate()
          .clusterId("foo")
          .resource(new JobDefinitionResource().vCores(1L).memory(2048L).executionSlots(1L));
      JobDefinition job = new JobDefinition()
          .query("SELECT * FROM input.foo")
          .addDesiredStateItem(state);
      api.updateJob(UUID.fromString(uuid), job);

      try (KafkaConsumer<byte[], byte[]> consumer = getConsumer("observer", brokerAddress())) {
        consumer.subscribe(Collections.singletonList(DEST_TOPIC));
        boolean found = false;
        while (!found) {
          ConsumerRecords<byte[], byte[]> records = consumer.poll(1000);
          for (ConsumerRecord<byte[], byte[]> r : records.records(DEST_TOPIC)) {
            @SuppressWarnings("unchecked")
            Map<String, Object> m = MAPPER.readValue(r.value(), Map.class);
            if ((Integer) m.get("id") == 2) {
              found = true;
            }
          }
        }
        ServerContext.INSTANCE.executor().shutdown();
        ServerContext.INSTANCE.instanceManager().close();
      }
    }
  }
}
 

@Override public ConsumerRecords<K, V> onConsume(ConsumerRecords<K, V> records) {
  if (records.isEmpty() || tracing.isNoop()) return records;
  Map<String, Span> consumerSpansForTopic = new LinkedHashMap<>();
  for (TopicPartition partition : records.partitions()) {
    String topic = partition.topic();
    List<ConsumerRecord<K, V>> recordsInPartition = records.records(partition);
    for (ConsumerRecord<K, V> record : recordsInPartition) {
      TraceContextOrSamplingFlags extracted = extractor.extract(record.headers());
      // If we extracted neither a trace context, nor request-scoped data
      // (extra),
      // make or reuse a span for this topic
      if (extracted.samplingFlags() != null && extracted.extra().isEmpty()) {
        Span consumerSpanForTopic = consumerSpansForTopic.get(topic);
        if (consumerSpanForTopic == null) {
          consumerSpansForTopic.put(topic,
            consumerSpanForTopic = tracing.tracer()
              .nextSpan(extracted)
              .name(SPAN_NAME)
              .kind(Span.Kind.CONSUMER)
              .remoteServiceName(remoteServiceName)
              .tag(KafkaInterceptorTagKey.KAFKA_TOPIC, topic)
              .tag(KafkaInterceptorTagKey.KAFKA_GROUP_ID,
                configuration.getString(ConsumerConfig.GROUP_ID_CONFIG))
              .tag(KafkaInterceptorTagKey.KAFKA_CLIENT_ID,
                configuration.getString(ConsumerConfig.CLIENT_ID_CONFIG))
              .start());
        }
        // no need to remove propagation headers as we failed to extract
        // anything
        injector.inject(consumerSpanForTopic.context(), record.headers());
      } else { // we extracted request-scoped data, so cannot share a consumer
        // span.
        Span span = tracing.tracer().nextSpan(extracted);
        if (!span.isNoop()) {
          span.name(SPAN_NAME)
            .kind(Span.Kind.CONSUMER)
            .remoteServiceName(remoteServiceName)
            .tag(KafkaInterceptorTagKey.KAFKA_TOPIC, topic)
            .tag(KafkaInterceptorTagKey.KAFKA_GROUP_ID,
              configuration.getString(ConsumerConfig.GROUP_ID_CONFIG))
            .tag(KafkaInterceptorTagKey.KAFKA_CLIENT_ID,
              configuration.getString(ConsumerConfig.CLIENT_ID_CONFIG))
            .start()
            .finish(); // span won't be shared by other records
        }
        // remove prior propagation headers from the record
        tracing.propagation().keys().forEach(key -> record.headers().remove(key));
        injector.inject(span.context(), record.headers());
      }
    }
  }
  consumerSpansForTopic.values().forEach(Span::finish);
  return records;
}
 

@Override
public void runFetchLoop() throws Exception {
	try {
		final Handover handover = this.handover;

		// kick off the actual Kafka consumer
		consumerThread.start();

		while (running) {
			// this blocks until we get the next records
			// it automatically re-throws exceptions encountered in the consumer thread
			final ConsumerRecords<byte[], byte[]> records = handover.pollNext();

			// get the records for each topic partition
			for (KafkaTopicPartitionState<TopicPartition> partition : subscribedPartitionStates()) {

				List<ConsumerRecord<byte[], byte[]>> partitionRecords =
					records.records(partition.getKafkaPartitionHandle());

				for (ConsumerRecord<byte[], byte[]> record : partitionRecords) {
					final T value = deserializer.deserialize(record);

					if (deserializer.isEndOfStream(value)) {
						// end of stream signaled
						running = false;
						break;
					}

					// emit the actual record. this also updates offset state atomically
					// and deals with timestamps and watermark generation
					emitRecord(value, partition, record.offset(), record);
				}
			}
		}
	}
	finally {
		// this signals the consumer thread that no more work is to be done
		consumerThread.shutdown();
	}

	// on a clean exit, wait for the runner thread
	try {
		consumerThread.join();
	}
	catch (InterruptedException e) {
		// may be the result of a wake-up interruption after an exception.
		// we ignore this here and only restore the interruption state
		Thread.currentThread().interrupt();
	}
}
 

@Override
public void runFetchLoop() throws Exception {
	try {
		final Handover handover = this.handover;

		// kick off the actual Kafka consumer
		consumerThread.start();

		while (running) {
			// this blocks until we get the next records
			// it automatically re-throws exceptions encountered in the consumer thread
			final ConsumerRecords<byte[], byte[]> records = handover.pollNext();

			// get the records for each topic partition
			for (KafkaTopicPartitionState<TopicPartition> partition : subscribedPartitionStates()) {

				List<ConsumerRecord<byte[], byte[]>> partitionRecords =
						records.records(partition.getKafkaPartitionHandle());

				for (ConsumerRecord<byte[], byte[]> record : partitionRecords) {

					final T value = deserializer.deserialize(record);

					if (deserializer.isEndOfStream(value)) {
						// end of stream signaled
						running = false;
						break;
					}

					// emit the actual record. this also updates offset state atomically
					// and deals with timestamps and watermark generation
					emitRecord(value, partition, record.offset(), record);
				}
			}
		}
	}
	finally {
		// this signals the consumer thread that no more work is to be done
		consumerThread.shutdown();
	}

	// on a clean exit, wait for the runner thread
	try {
		consumerThread.join();
	}
	catch (InterruptedException e) {
		// may be the result of a wake-up interruption after an exception.
		// we ignore this here and only restore the interruption state
		Thread.currentThread().interrupt();
	}
}
 

@Override
public void runFetchLoop() throws Exception {
	try {
		final Handover handover = this.handover;

		// kick off the actual Kafka consumer
		consumerThread.start();

		while (running) {
			// this blocks until we get the next records
			// it automatically re-throws exceptions encountered in the consumer thread
			final ConsumerRecords<byte[], byte[]> records = handover.pollNext();

			// get the records for each topic partition
			for (KafkaTopicPartitionState<TopicPartition> partition : subscribedPartitionStates()) {

				List<ConsumerRecord<byte[], byte[]>> partitionRecords =
						records.records(partition.getKafkaPartitionHandle());

				for (ConsumerRecord<byte[], byte[]> record : partitionRecords) {

					final T value = deserializer.deserialize(record);

					if (deserializer.isEndOfStream(value)) {
						// end of stream signaled
						running = false;
						break;
					}

					// emit the actual record. this also updates offset state atomically
					// and deals with timestamps and watermark generation
					emitRecord(value, partition, record.offset(), record);
				}
			}
		}
	}
	finally {
		// this signals the consumer thread that no more work is to be done
		consumerThread.shutdown();
	}

	// on a clean exit, wait for the runner thread
	try {
		consumerThread.join();
	}
	catch (InterruptedException e) {
		// may be the result of a wake-up interruption after an exception.
		// we ignore this here and only restore the interruption state
		Thread.currentThread().interrupt();
	}
}
 

public static void main(String[] args) {

        KafkaConsumer<String, String> consumer = createKafkaConsumer();
        KafkaProducer<String, String> producer = createKafkaProducer();

        producer.initTransactions();

        try {

            while (true) {

                ConsumerRecords<String, String> records = consumer.poll(ofSeconds(60));

                Map<String, Integer> wordCountMap = records.records(new TopicPartition(INPUT_TOPIC, 0))
                        .stream()
                        .flatMap(record -> Stream.of(record.value().split(" ")))
                        .map(word -> Tuple.of(word, 1))
                        .collect(Collectors.toMap(tuple -> tuple.getKey(), t1 -> t1.getValue(), (v1, v2) -> v1 + v2));

                producer.beginTransaction();

                wordCountMap.forEach((key, value) -> producer.send(new ProducerRecord<String, String>(OUTPUT_TOPIC, key, value.toString())));

                Map<TopicPartition, OffsetAndMetadata> offsetsToCommit = new HashMap<>();

                for (TopicPartition partition : records.partitions()) {
                    List<ConsumerRecord<String, String>> partitionedRecords = records.records(partition);
                    long offset = partitionedRecords.get(partitionedRecords.size() - 1).offset();

                    offsetsToCommit.put(partition, new OffsetAndMetadata(offset + 1));
                }

                producer.sendOffsetsToTransaction(offsetsToCommit, CONSUMER_GROUP_ID);
                producer.commitTransaction();

            }

        } catch (KafkaException e) {

            producer.abortTransaction();

        }


    }