kafka.common.OffsetAndMetadata Java Examples

/**
 * When an object implementing interface <code>Runnable</code> is used
 * to create a thread, starting the thread causes the object's
 * <code>run</code> method to be called in that separately executing
 * thread.
 * <p>
 * The general contract of the method <code>run</code> is that it may
 * take any action whatsoever.
 *
 * @see Thread#run()
 */
@Override
public void run() {
    ConsumerConnector consumerConnector = KafkaUtils.createConsumerConnector(zkAddr, group);
    Map<String, Integer> topicCountMap = new HashMap<String, Integer>();
    topicCountMap.put(CONSUMER_OFFSET_TOPIC, new Integer(1));
    KafkaStream<byte[], byte[]> offsetMsgStream = consumerConnector.createMessageStreams(topicCountMap).get(CONSUMER_OFFSET_TOPIC).get(0);

    ConsumerIterator<byte[], byte[]> it = offsetMsgStream.iterator();
    while (true) {

        MessageAndMetadata<byte[], byte[]> offsetMsg = it.next();
        if (ByteBuffer.wrap(offsetMsg.key()).getShort() < 2) {
            try {
                GroupTopicPartition commitKey = readMessageKey(ByteBuffer.wrap(offsetMsg.key()));
                if (offsetMsg.message() == null) {
                    continue;
                }
                kafka.common.OffsetAndMetadata commitValue = readMessageValue(ByteBuffer.wrap(offsetMsg.message()));
                kafkaConsumerOffsets.put(commitKey, commitValue);
            } catch (Exception e) {
                e.printStackTrace();
            }
        }
    }
}
 

public static void main(String[] args) {
		Properties prop = new Properties();
		prop.put("group.id", "kafka.eagle.system.group");
		prop.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, "127.0.0.1:9092");
		prop.put("exclude.internal.topics", "false");
		prop.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getCanonicalName());
		prop.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getCanonicalName());
		KafkaConsumer<String, String> consumer = new KafkaConsumer<>(prop);
		consumer.subscribe(Arrays.asList("__consumer_offsets"));
		boolean flag = true;
		while (flag) {
			try {
				ConsumerRecords<String, String> records = consumer.poll(Duration.ofMillis(100));
				for (ConsumerRecord<String, String> record : records) {
					ByteBuffer buffer = ByteBuffer.wrap(record.value().getBytes());
					
//					OffsetAndMetadata meta = GroupMetadataManager.readOffsetMessageValue();
//					BaseKey key = readMessageKey(buffer);
					OffsetAndMetadata gm = readOffsetMessageValue(buffer);
					System.out.println(gm);
				}
			}catch (Exception e) {
				// TODO: handle exception
				e.printStackTrace();
			}
		}
	}
 

private static synchronized void startKafkaOffsetListener(String clusterAlias, KafkaConsumer<String, String> consumer) {
	consumer.subscribe(Arrays.asList(CONSUMER_OFFSET_TOPIC));
	boolean flag = true;
	while (flag) {
		try {
			ConsumerRecords<String, String> records = consumer.poll(Duration.ofMillis(100));
			for (ConsumerRecord<String, String> record : records) {
				try {
					if (record != null && record.value() != null) {
						GroupTopicPartition offsetKey = readMessageKey(ByteBuffer.wrap(record.key().getBytes()));
						if (offsetKey != null) {
							GroupTopicPartition commitKey = offsetKey;
							if (commitKey.topicPartition().topic().equals(CONSUMER_OFFSET_TOPIC)) {
								continue;
							}

							OffsetAndMetadata commitValue = readOffsetMessageValue(ByteBuffer.wrap(record.value().getBytes()));
							if (multiKafkaConsumerOffsets.containsKey(clusterAlias)) {
								multiKafkaConsumerOffsets.get(clusterAlias).put(commitKey, commitValue);
							} else {
								Map<GroupTopicPartition, OffsetAndMetadata> kafkaConsumerOffsets = new ConcurrentHashMap<>();
								kafkaConsumerOffsets.put(commitKey, commitValue);
								multiKafkaConsumerOffsets.put(clusterAlias, kafkaConsumerOffsets);
							}
						}
						System.out.println(multiKafkaConsumerOffsets.toString());
					}
				} catch (Exception e) {
					LOG.error("Get consumer records has error, msg is " + e.getMessage());
				}
			}
		} catch (Exception ex) {
			LOG.error("Start kafka sasl listener has error, msg is " + ex.getMessage());
		}
	}
}
 

@Override
public void run() {

    logger.info(Thread.currentThread().getName() + " clientId: "
            + stream.clientId() + " start.");

    ConsumerIterator<K, V> it = stream.iterator();

    while (it.hasNext()) {

        MessageAndMetadata<K, V> messageAndMetadata = it.next();

        try {
            this.adapter.messageAdapter(messageAndMetadata);

        } catch (MQException e) {

            if (receiverRetry != null)

                receiverRetry.receiveMessageRetry(messageAndMetadata);

            logger.error("Receive message failed."
                    + " topic: " + messageAndMetadata.topic()
                    + " offset: " + messageAndMetadata.offset()
                    + " partition: " + messageAndMetadata.partition(), e);

        } finally {

            /* commitOffsets */
            if (!getAutoCommit()) {
                consumer.commitOffsets(Collections.singletonMap(
                        TopicAndPartition.apply(messageAndMetadata.topic(), messageAndMetadata.partition()),
                        OffsetAndMetadata.apply(messageAndMetadata.offset() + 1)), true);
            }
        }
    }

    logger.info(Thread.currentThread().getName() + " clientId: " + stream.clientId() + " end.");
}
 

@Override
public Short execute() throws Exception {
  short versionID = 0;
  int correlationId = 0;
  TopicAndPartition tp = new TopicAndPartition(topic, partitionMetadata.partitionId());
  OffsetAndMetadata offsetAndMeta = new OffsetAndMetadata(offset, OffsetAndMetadata.NoMetadata(), errorCode);
  Map<TopicAndPartition, OffsetAndMetadata> mapForCommitOffset = new HashMap<TopicAndPartition, OffsetAndMetadata>();
  mapForCommitOffset.put(tp, offsetAndMeta);
  OffsetCommitRequest offsetCommitReq = new OffsetCommitRequest(name, mapForCommitOffset, correlationId, name, versionID);
  OffsetCommitResponse offsetCommitResp = consumer.commitOffsets(offsetCommitReq);
  return (Short) offsetCommitResp.errors().get(tp);
}
 

/**
     * 更新偏移量，当SimpleConsumer发生变化的时候，重新构造一个新的SimpleConsumer并返回
     *
     * @param consumer
     * @param topic
     * @param partitionID
     * @param readOffSet
     * @param groupId
     * @param clientName
     * @param times
     * @return
     * @throws RuntimeException 当更新失败的情况下
     */
    private SimpleConsumer updateOffset(SimpleConsumer consumer, String topic, int partitionID, long readOffSet, String groupId, String clientName, int times) {
        // 构建请求对象
        Map<TopicAndPartition, OffsetAndMetadata> requestInfoMap = new HashMap<TopicAndPartition, OffsetAndMetadata>();
        TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partitionID);
//        requestInfoMap.put(topicAndPartition, new OffsetAndMetadata(readOffSet, OffsetAndMetadata.NoMetadata(), -1));
        kafka.javaapi.OffsetCommitRequest ocRequest = new OffsetCommitRequest(groupId, requestInfoMap, 0, clientName);
        // 提交修改偏移量的请求，并获取返回值
        kafka.javaapi.OffsetCommitResponse response = consumer.commitOffsets(ocRequest);

        // 根据返回值进行不同的操作
        if (response.hasError()) {
            short code = response.errorCode(topicAndPartition);
            if (times > this.maxRetryTimes) {
                throw new RuntimeException("Update the Offset occur exception," +
                        " the current response code is:" + code);
            }

            if (code == ErrorMapping.LeaderNotAvailableCode()) {
                // 当异常code为leader切换情况的时候，重新构建consumer对象
                // 操作步骤：先休眠一段时间，再重新构造consumer对象，最后重试
                try {
                    Thread.sleep(this.retryIntervalMillis);
                } catch (InterruptedException e) {
                    // nothings
                }
                PartitionMetadata metadata = this.findNewLeaderMetadata(consumer.host(),
                        topic, partitionID);
                this.validatePartitionMetadata(metadata);
                consumer = this.createSimpleConsumer(metadata.leader().host(),
                        metadata.leader().port(), clientName);
                // 重试
                consumer = updateOffset(consumer, topic, partitionID, readOffSet, groupId, clientName, times + 1);
            }

            if (code == ErrorMapping.RequestTimedOutCode()) {
                // 当异常为请求超时的时候，进行重新请求
                consumer = updateOffset(consumer, topic, partitionID, readOffSet, groupId, clientName, times + 1);
            }

            // 其他code直接抛出异常
            throw new RuntimeException("Update the Offset occur exception," +
                    " the current response code is:" + code);
        }

        // 返回修改后的consumer对象
        return consumer;
    }
 

public static List<OffsetInfo> getOffsetQuarz() {

        Map<String, Map<String, List<OffsetInfo>>> groupTopicPartitionListMap = new ConcurrentHashMap<>();

        for (Map.Entry<GroupTopicPartition, OffsetAndMetadata> entry: kafkaConsumerOffsets.entrySet()) {
            GroupTopicPartition groupTopicPartition = entry.getKey();
            OffsetAndMetadata offsetAndMetadata = entry.getValue();
            String group = groupTopicPartition.group();
            TopicPartition topicPartition = groupTopicPartition.topicPartition();
            String topic = topicPartition.topic();
            int partition = topicPartition.partition();
            Long committedOffset = offsetAndMetadata.offset();

            if (!logEndOffsetMap.containsKey(topicPartition)) {
                logger.error("The logEndOffsetMap not contains " + topicPartition);
                return null;
            }
            long logSize = logEndOffsetMap.get(topicPartition);

            // May the refresh operation thread take some time to update
            logSize = logSize >= committedOffset ? logSize : committedOffset;
            long lag = committedOffset == -1 ? 0 : (logSize - committedOffset);

            OffsetInfo offsetInfo = new OffsetInfo();
            offsetInfo.setGroup(group);
            offsetInfo.setTopic(topic);
            offsetInfo.setCommittedOffset(committedOffset);
            offsetInfo.setLogSize(logSize);
            offsetInfo.setLag(lag);
            offsetInfo.setTimestamp(offsetAndMetadata.commitTimestamp());

            if (!groupTopicPartitionListMap.containsKey(group)) {
                Map<String, List<OffsetInfo>> topicPartitionMap = new ConcurrentHashMap<>();
                groupTopicPartitionListMap.put(group, topicPartitionMap);
            }
            if (!groupTopicPartitionListMap.get(group).containsKey(topic)) {
                List<OffsetInfo> offsetInfos = new ArrayList<>();
                groupTopicPartitionListMap.get(group).put(topic, offsetInfos);
            }
            groupTopicPartitionListMap.get(group).get(topic).add(offsetInfo);

        }
        return flattenNestedMap(groupTopicPartitionListMap);
    }