org.apache.kafka.clients.consumer.ConsumerRecord#key

Source File: BaseKafkaConsumer11.java From datacollector with Apache License 2.0

6 votes

@Override
MessageAndOffset getMessageAndOffset(ConsumerRecord message, boolean isEnabled) {
  MessageAndOffset messageAndOffset;
  if (message.timestampType() != TimestampType.NO_TIMESTAMP_TYPE && message.timestamp() > 0 && isEnabled) {
    messageAndOffset = new MessageAndOffsetWithTimestamp(
        message.key(),
        message.value(),
        message.offset(),
        message.partition(),
        message.timestamp(),
        message.timestampType().toString()
    );
  } else {
    messageAndOffset = new MessageAndOffset(message.key(), message.value(), message.offset(), message.partition());
  }
  return messageAndOffset;
}

Source File: ParallelWebKafkaConsumer.java From kafka-webview with MIT License

6 votes

private List<KafkaResult> consume(final KafkaConsumer kafkaConsumer) {
    final List<KafkaResult> kafkaResultList = new ArrayList<>();
    final ConsumerRecords<?,?> consumerRecords = kafkaConsumer.poll(pollTimeoutDuration);

    logger.info("Consumed {} records", consumerRecords.count());
    for (final ConsumerRecord consumerRecord : consumerRecords) {
        // Get next record
        // Convert to KafkaResult.
        final KafkaResult kafkaResult = new KafkaResult(
            consumerRecord.partition(),
            consumerRecord.offset(),
            consumerRecord.timestamp(),
            consumerRecord.key(),
            consumerRecord.value()
        );

        // Add to list.
        kafkaResultList.add(kafkaResult);
    }

    // Commit offsets
    commit(kafkaConsumer);
    return kafkaResultList;
}

Source File: KafkaConsumerHelper.java From zerocode with Apache License 2.0

6 votes

public static void readJson(List<ConsumerJsonRecord> jsonRecords,
                            Iterator recordIterator) throws IOException {
    while (recordIterator.hasNext()) {
        ConsumerRecord thisRecord = (ConsumerRecord) recordIterator.next();

        Object key = thisRecord.key();
        Object value = thisRecord.value();
        Headers headers = thisRecord.headers();
        LOGGER.info("\nRecord Key - {} , Record value - {}, Record partition - {}, Record offset - {}, Headers - {}",
                key, value, thisRecord.partition(), thisRecord.offset(), headers);

        JsonNode valueNode = objectMapper.readTree(value.toString());
        Map<String, String> headersMap = null;
        if (headers != null) {
            headersMap = new HashMap<>();
            for (Header header : headers) {
                headersMap.put(header.key(), new String(header.value()));
            }
        }
        ConsumerJsonRecord jsonRecord = new ConsumerJsonRecord(thisRecord.key(), null, valueNode, headersMap);
        jsonRecords.add(jsonRecord);
    }
}

Source File: ConsumeDataIterator.java From oryx with Apache License 2.0

6 votes

@Override
protected KeyMessage<K,V> computeNext() {
  if (iterator == null || !iterator.hasNext()) {
    try {
      long timeout = MIN_POLL_MS;
      ConsumerRecords<K, V> records;
      while ((records = consumer.poll(timeout)).isEmpty()) {
        timeout = Math.min(MAX_POLL_MS, timeout * 2);
      }
      iterator = records.iterator();
    } catch (Exception e) {
      consumer.close();
      return endOfData();
    }
  }
  ConsumerRecord<K,V> mm = iterator.next();
  return new KeyMessageImpl<>(mm.key(), mm.value());
}

Source File: TopicFilter.java From arcusplatform with Apache License 2.0

6 votes

@Override
public void onMessage(ConsumerRecord<K, byte[]> record) {
   K key = record.key();
   if (key == null && !acceptNullKeys()) {
      return;
   }
   if (!acceptKey(key)) {
      return;
   }

   M payload = deserializeMessage(record.value());
   if (!acceptMessage(payload)) {
      return;
   }
   deliver(payload);
}

Source File: SecorKafkaClient.java From secor with Apache License 2.0

6 votes

private Message readSingleMessage(KafkaConsumer<byte[], byte[]> kafkaConsumer) {
    int pollAttempts = 0;
    Message message = null;
    while (pollAttempts < MAX_READ_POLL_ATTEMPTS) {
        Iterator<ConsumerRecord<byte[], byte[]>> records = kafkaConsumer.poll(Duration.ofSeconds(mPollTimeout)).iterator();
        if (!records.hasNext()) {
            pollAttempts++;
        } else {
            ConsumerRecord<byte[], byte[]> record = records.next();
            List<MessageHeader> headers = new ArrayList<>();
            record.headers().forEach(header -> headers.add(new MessageHeader(header.key(), header.value())));
            message = new Message(record.topic(), record.partition(), record.offset(), record.key(), record.value(), record.timestamp(), headers);
            break;
        }
    }

    if (message == null) {
        LOG.warn("unable to fetch message after " + MAX_READ_POLL_ATTEMPTS + " Retries");
    }
    return message;
}

Source File: KafkaConsumer10.java From datacollector with Apache License 2.0

6 votes

@Override
MessageAndOffset getMessageAndOffset(ConsumerRecord message, boolean isEnabled) {
  MessageAndOffset messageAndOffset;
  if (message.timestampType() != TimestampType.NO_TIMESTAMP_TYPE && message.timestamp() > 0 && isEnabled) {
    messageAndOffset = new MessageAndOffsetWithTimestamp(
        message.key(),
        message.value(),
        message.offset(),
        message.partition(),
        message.timestamp(),
        message.timestampType().toString()
    );
  } else {
    messageAndOffset = new MessageAndOffset(message.key(), message.value(), message.offset(), message.partition());
  }
  return messageAndOffset;
}

Source File: DefaultWebKafkaConsumer.java From kafka-webview with MIT License

5 votes

private List<KafkaResult> consume() {
    final List<KafkaResult> kafkaResultList = new ArrayList<>();
    final ConsumerRecords consumerRecords = kafkaConsumer.poll(pollTimeoutDuration);

    logger.info("Consumed {} records", consumerRecords.count());
    final Iterator<ConsumerRecord> recordIterator = consumerRecords.iterator();
    while (recordIterator.hasNext()) {
        // Get next record
        final ConsumerRecord consumerRecord = recordIterator.next();

        // Convert to KafkaResult.
        final KafkaResult kafkaResult = new KafkaResult(
            consumerRecord.partition(),
            consumerRecord.offset(),
            consumerRecord.timestamp(),
            consumerRecord.key(),
            consumerRecord.value()
        );

        // Add to list.
        kafkaResultList.add(kafkaResult);
    }

    // Commit offsets
    commit();
    return kafkaResultList;
}

Source File: KafkaEasyTransMsgConsumerImpl.java From EasyTransaction with Apache License 2.0

5 votes

private void reconsumeLater(ConsumerRecord<String, byte[]> consumeRecord) throws InterruptedException, ExecutionException {

		// add all header to headList except RETRY_COUNT
		Headers headers = consumeRecord.headers();
		List<Header> headerList = new ArrayList<Header>(8);
		Iterator<Header> iterator = headers.iterator();
		Integer retryCount = -1;
		boolean hasOrignalHeader = false;
		while (iterator.hasNext()) {
			Header next = iterator.next();
			if (next.key().equals(RETRY_COUNT_KEY)) {
				retryCount = serializer.deserialize(next.value());
				continue;
			}
			
			if(next.key().equals(ORGINAL_TOPIC)){
				hasOrignalHeader = true;
			}
			headerList.add(next);
		}
		
		// add RETRY_COUNT to header
		retryCount++;
		headerList.add(new RecordHeader(RETRY_COUNT_KEY, serializer.serialization(retryCount)));
		
		if(!hasOrignalHeader){
			headerList.add(new RecordHeader(ORGINAL_TOPIC, serializer.serialization(consumeRecord.topic())));
		}

		// send message to corresponding queue according to retry times
		String retryTopic = calcRetryTopic(consumeRecord.topic(), retryCount);
		
		ProducerRecord<String, byte[]> record = new ProducerRecord<>(retryTopic,
				consumeRecord.partition() % retryQueuePartitionCount.get(retryTopic), null, consumeRecord.key(),
				consumeRecord.value(), headerList);
		Future<RecordMetadata> publishKafkaMessage = retryQueueMsgProducer.publishKafkaMessage(record);
		publishKafkaMessage.get();
	}

Source File: KafkaInput.java From envelope with Apache License 2.0

5 votes

@Override
public Row call(ConsumerRecord record) {
  return new RowWithSchema(
      getProvidingSchema(),
      record.key(),
      record.value(),
      record.timestamp(),
      record.topic(),
      record.partition(),
      record.offset());
}

Source File: KafkaTracing.java From brave with Apache License 2.0

5 votes

/** When an upstream context was not present, lookup keys are unlikely added */
static void addTags(ConsumerRecord<?, ?> record, SpanCustomizer result) {
  if (record.key() instanceof String && !"".equals(record.key())) {
    result.tag(KafkaTags.KAFKA_KEY_TAG, record.key().toString());
  }
  result.tag(KafkaTags.KAFKA_TOPIC_TAG, record.topic());
}

Source File: TypeInformationKeyValueSerializationSchema.java From flink with Apache License 2.0

5 votes

@Override
public Tuple2<K, V> deserialize(ConsumerRecord<byte[], byte[]> record) throws Exception {
	K key = null;
	V value = null;

	if (record.key() != null) {
		inputDeserializer.setBuffer(record.key());
		key = keySerializer.deserialize(inputDeserializer);
	}
	if (record.value() != null) {
		inputDeserializer.setBuffer(record.value());
		value = valueSerializer.deserialize(inputDeserializer);
	}
	return new Tuple2<>(key, value);
}

Source File: DistributedClusterTest.java From kop with Apache License 2.0

5 votes

protected void kafkaConsumeCommitMessage(KConsumer kConsumer,
                                         int numMessages,
                                         String messageStrPrefix,
                                         List<TopicPartition> topicPartitions) {
    kConsumer.getConsumer().assign(topicPartitions);
    int i = 0;
    while (i < numMessages) {
        if (log.isDebugEnabled()) {
            log.debug("kConsumer {} start poll message: {}",
                kConsumer.getTopic() + kConsumer.getConsumerGroup(), i);
        }
        ConsumerRecords<Integer, String> records = kConsumer.getConsumer().poll(Duration.ofSeconds(1));
        for (ConsumerRecord<Integer, String> record : records) {
            Integer key = record.key();
            assertEquals(messageStrPrefix + key.toString(), record.value());

            if (log.isDebugEnabled()) {
                log.debug("Kafka consumer get message: {}, key: {} at offset {}",
                    record.key(), record.value(), record.offset());
            }
            i++;
        }
    }

    assertEquals(i, numMessages);

    try {
        kConsumer.getConsumer().commitSync(Duration.ofSeconds(1));
    } catch (Exception e) {
        log.error("Commit offset failed: ", e);
    }

    if (log.isDebugEnabled()) {
        log.debug("kConsumer {} finished poll and commit message: {}",
            kConsumer.getTopic() + kConsumer.getConsumerGroup(), i);
    }
}

Source File: KafkaUnit.java From SkaETL with Apache License 2.0

4 votes

@Override
public Message<String, String> extract(final ConsumerRecord<byte[], byte[]> record) {
    String key = record.key() != null ? new String(record.key()) : null;
    String value = record.value() != null ? new String(record.value()) : null;
    return new Message<>(key, value);
}

Source File: ProductListener.java From micronaut-kafka with Apache License 2.0

4 votes

@Topic("awesome-products")
public void receive(ConsumerRecord<String, Product> record) { // <1>
    Product product = record.value(); // <2>
    String brand = record.key(); // <3>
    System.out.println("Got Product - " + product.getName() + " by " + brand);
}

Source File: KafkaKeyBinder.java From micronaut-kafka with Apache License 2.0

4 votes

@Override
public BindingResult<T> bind(ArgumentConversionContext<T> context, ConsumerRecord<?, ?> source) {
    Object key = source.key();
    Optional<T> converted = ConversionService.SHARED.convert(key, context);
    return () -> converted;
}

Source File: KafkaConsumer09.java From datacollector with Apache License 2.0

4 votes

@Override
MessageAndOffset getMessageAndOffset(ConsumerRecord message, boolean isEnabled) {
  return new MessageAndOffset(message.key(), message.value(), message.offset(), message.partition());
}

Source File: NewApiTopicConsumer.java From jeesuite-libs with Apache License 2.0

4 votes

/**
 * @param record
 */
private void processConsumerRecords(final ConsumerRecord<String, Serializable> record) {
	//兼容没有包装的情况
	final DefaultMessage message = record.value() instanceof DefaultMessage ? (DefaultMessage) record.value() : new DefaultMessage(record.key(),(Serializable) record.value());
	final MessageHandler messageHandler = topicHandlers.get(record.topic());
	
	message.setTopicMetadata(record.topic(), record.partition(), record.offset());
	
	consumerContext.updateConsumerStats(record.topic(),1);
	//
	consumerContext.saveOffsetsBeforeProcessed(record.topic(), record.partition(), record.offset() + 1);
	//第一阶段处理
	messageHandler.p1Process(message);
	//第二阶段处理
	(message.isConsumerAckRequired() ? highProcessExecutor : defaultProcessExecutor).submit(new Runnable() {
		@Override
		public void run() {
			try {									
				messageHandler.p2Process(message);
				//
				if(!offsetAutoCommit){
					uncommittedOffsetMap.put(new TopicPartition(record.topic(), record.partition()), new OffsetAndMetadata(record.offset() + 1));
					//
					uncommittedNums.incrementAndGet();
				}
				
				//回执
                      if(message.isConsumerAckRequired()){
                      	consumerContext.sendConsumerAck(message.getMsgId());
				}
				//
				consumerContext.saveOffsetsAfterProcessed(record.topic(), record.partition(), record.offset() + 1);
			} catch (Exception e) {
				boolean processed = messageHandler.onProcessError(message);
				if(processed == false){
					consumerContext.processErrorMessage(record.topic(), message);
				}
				logger.error("["+messageHandler.getClass().getSimpleName()+"] process Topic["+record.topic()+"] error",e);
			}
			
			consumerContext.updateConsumerStats(record.topic(),-1);
		}
	});
}

Source File: DBusConsumerRecord.java From DBus with Apache License 2.0

4 votes

public DBusConsumerRecord(ConsumerRecord<K, V> record) {
    this(record.topic(), record.partition(), record.offset(), record.timestamp(), record.timestampType(),
            record.checksum(), record.serializedKeySize(), record.serializedValueSize(), record.key(), record.value());
}

Source File: KafkaConsumerProxy.java From samza with Apache License 2.0

2 votes

/**
 * Protected to help extensions of this class build {@link IncomingMessageEnvelope}s.
 * @param r consumer record to size
 * @return the size of the serialized record
 */
protected int getRecordSize(ConsumerRecord<K, V> r) {
  int keySize = (r.key() == null) ? 0 : r.serializedKeySize();
  return keySize + r.serializedValueSize();
}

Java Code Examples for org.apache.kafka.clients.consumer.ConsumerRecord#key()