org.apache.kafka.common.header.Header Java Examples

public static void readJson(List<ConsumerJsonRecord> jsonRecords,
                            Iterator recordIterator) throws IOException {
    while (recordIterator.hasNext()) {
        ConsumerRecord thisRecord = (ConsumerRecord) recordIterator.next();

        Object key = thisRecord.key();
        Object value = thisRecord.value();
        Headers headers = thisRecord.headers();
        LOGGER.info("\nRecord Key - {} , Record value - {}, Record partition - {}, Record offset - {}, Headers - {}",
                key, value, thisRecord.partition(), thisRecord.offset(), headers);

        JsonNode valueNode = objectMapper.readTree(value.toString());
        Map<String, String> headersMap = null;
        if (headers != null) {
            headersMap = new HashMap<>();
            for (Header header : headers) {
                headersMap.put(header.key(), new String(header.value()));
            }
        }
        ConsumerJsonRecord jsonRecord = new ConsumerJsonRecord(thisRecord.key(), null, valueNode, headersMap);
        jsonRecords.add(jsonRecord);
    }
}
 

private static Header[] getHeadersFromMetadata(List<KeyValue> properties) {
    Header[] headers = new Header[properties.size()];

    if (log.isDebugEnabled()) {
        log.debug("getHeadersFromMetadata. Header size: {}",
            properties.size());
    }

    int index = 0;
    for (KeyValue kv: properties) {
        headers[index] = new RecordHeader(kv.getKey(), kv.getValue().getBytes(UTF_8));

        if (log.isDebugEnabled()) {
            log.debug("index: {} kv.getKey: {}. kv.getValue: {}",
                index, kv.getKey(), kv.getValue());
        }
        index++;
    }

    return headers;
}
 

@Test
public void before() {

    doReturn(trace).when(traceContext).currentRawTraceObject();
    doReturn(true).when(trace).canSampled();
    doReturn(traceId).when(trace).getTraceId();
    doReturn(nextId).when(traceId).getNextTraceId();
    doReturn(recorder).when(trace).traceBlockBegin();
    doReturn(headers).when(record).headers();
    doReturn(new Header[]{}).when(headers).toArray();
    doReturn("test").when(nextId).getTransactionId();
    doReturn(0l).when(nextId).getSpanId();
    doReturn(0l).when(nextId).getParentSpanId();
    short s = 0;
    doReturn(s).when(nextId).getFlags();

    ProducerSendInterceptor interceptor = new ProducerSendInterceptor(traceContext, descriptor);
    Object target = new Object();
    Object[] args = new Object[]{record};
    interceptor.before(target, args);

    verify(recorder).recordServiceType(KafkaConstants.KAFKA_CLIENT);

}
 

private Map<String, String> getAttributes(final ConsumerRecord<?, ?> consumerRecord) {
    final Map<String, String> attributes = new HashMap<>();
    if (headerNamePattern == null) {
        return attributes;
    }

    for (final Header header : consumerRecord.headers()) {
        final String attributeName = header.key();
        final byte[] attributeValue = header.value();
        if (headerNamePattern.matcher(attributeName).matches() && attributeValue != null) {
            attributes.put(attributeName, new String(attributeValue, headerCharacterSet));
        }
    }

    return attributes;
}
 

@Test
public void createTrace() {

    doReturn(trace).when(traceContext).newTraceObject();
    doReturn(true).when(trace).canSampled();
    doReturn(recorder).when(trace).getSpanRecorder();

    doReturn("Test").when(consumerRecord).topic();
    doReturn(1l).when(consumerRecord).offset();
    doReturn(0).when(consumerRecord).partition();
    doReturn(headers).when(consumerRecord).headers();
    doReturn(new Header[]{}).when(headers).toArray();

    ConsumerRecordEntryPointInterceptor interceptor = new ConsumerRecordEntryPointInterceptor(traceContext, descriptor, 0);

    interceptor.createTrace(new Object(), new Object[]{consumerRecord});

    verify(recorder).recordAcceptorHost("Unknown");
    verify(recorder).recordRpcName("kafka://topic=Test?partition=0&offset=1");

}
 

@Test
public void testRecordHeaderSetWhilePublishing() {
    final String myTopic = "event-owner-selector-events";
    final BatchItem item = new BatchItem("{}", null,
            null,
            Collections.emptyList());
    item.setPartition("1");
    item.setOwner(new EventOwnerHeader("retailer", "nakadi"));
    final List<BatchItem> batch = ImmutableList.of(item);

    when(kafkaProducer.partitionsFor(myTopic)).thenReturn(ImmutableList.of(
            new PartitionInfo(myTopic, 1, NODE, null, null)));

    try {
        kafkaTopicRepository.syncPostBatch(myTopic, batch, "random", false);
        fail();
    } catch (final EventPublishingException e) {
        final ProducerRecord<String, String> recordSent = captureProducerRecordSent();
        final Header nameHeader = recordSent.headers().headers(EventOwnerHeader.AUTH_PARAM_NAME)
                .iterator().next();
        Assert.assertEquals(new String(nameHeader.value()), "retailer");
        final Header valueHeader = recordSent.headers().headers(EventOwnerHeader.AUTH_PARAM_VALUE)
                .iterator().next();
        Assert.assertEquals(new String(valueHeader.value()), "nakadi");
    }
}
 

/**
 * Gets the message type from the headers.  Throws if not found.
 *
 * @param headers the headers
 */
@SuppressWarnings("unchecked")
protected Class<T> getMessageType(Headers headers) {
    Header header = headers.lastHeader(JsonSchemaSerDeConstants.HEADER_MSG_TYPE);
    if (header == null) {
        throw new RuntimeException("Message Type not found in headers.");
    }
    String msgTypeName = IoUtil.toString(header.value());
    
    try {
        return (Class<T>) Thread.currentThread().getContextClassLoader().loadClass(msgTypeName);
    } catch (ClassNotFoundException ignored) {
    }
    try {
        return (Class<T>) Class.forName(msgTypeName);
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}
 

private Map<String, String> getAttributes(final ConsumerRecord<?, ?> consumerRecord) {
    final Map<String, String> attributes = new HashMap<>();
    if (headerNamePattern == null) {
        return attributes;
    }

    for (final Header header : consumerRecord.headers()) {
        final String attributeName = header.key();
        final byte[] attributeValue = header.value();
        if (headerNamePattern.matcher(attributeName).matches() && attributeValue != null) {
            attributes.put(attributeName, new String(attributeValue, headerCharacterSet));
        }
    }

    return attributes;
}
 

private Map<String, String> getAttributes(final ConsumerRecord<?, ?> consumerRecord) {
    final Map<String, String> attributes = new HashMap<>();
    if (headerNamePattern == null) {
        return attributes;
    }

    for (final Header header : consumerRecord.headers()) {
        final String attributeName = header.key();
        final byte[] attributeValue = header.value();
        if (headerNamePattern.matcher(attributeName).matches() && attributeValue != null) {
            attributes.put(attributeName, new String(attributeValue, headerCharacterSet));
        }
    }

    return attributes;
}
 

private boolean headerEqualityByValue(Header a, Header b) {
    // This is an alternative implementation of ConnectHeader::equals that use proper Value equality by value
    // (even if they are byte arrays)
    if (a == b) {
        return true;
    }
    if (!Objects.equals(a.key(), b.key())) {
        return false;
    }
    try {
        // This particular case is not handled by ConnectHeader::equals
        byte[] aBytes = a.value();
        byte[] bBytes = b.value();
        return Arrays.equals(aBytes, bBytes);
    } catch (ClassCastException e) {
        return a.value() == b.value();
    }
}
 

@SuppressWarnings("unchecked")
@Nullable
private Map<String, String> decodeJsonTypes(Headers source) {
	Map<String, String> types = null;
	Header jsonTypes = source.lastHeader(JSON_TYPES);
	if (jsonTypes != null) {
		ObjectMapper headerObjectMapper = getObjectMapper();
		try {
			types = headerObjectMapper.readValue(jsonTypes.value(), Map.class);
		}
		catch (IOException e) {
			logger.error(e, () -> "Could not decode json types: " + new String(jsonTypes.value()));
		}
	}
	return types;
}
 

private Object decodeValue(Header h, Class<?> type) throws IOException, LinkageError {
	ObjectMapper headerObjectMapper = getObjectMapper();
	Object value = headerObjectMapper.readValue(h.value(), type);
	if (type.equals(NonTrustedHeaderType.class)) {
		// Upstream NTHT propagated; may be trusted here...
		NonTrustedHeaderType nth = (NonTrustedHeaderType) value;
		if (trusted(nth.getUntrustedType())) {
			try {
				value = headerObjectMapper.readValue(nth.getHeaderValue(),
						ClassUtils.forName(nth.getUntrustedType(), null));
			}
			catch (Exception e) {
				logger.error(e, () -> "Could not decode header: " + nth);
			}
		}
	}
	return value;
}
 

/**
 * Special header keys have a "_" prefix and are managed internally by the clients.
 * @param headers kafka headers object
 * @return any "special" headers container in the argument map
 */
public static Map<String, byte[]> fetchSpecialHeaders(Headers headers) {
  Map<String, byte[]> map = new HashMap<>();
  for (Header header : headers) {

    if (!header.key().startsWith("_")) {
      // skip any non special header
      continue;
    }

    if (map.containsKey(header.key())) {
      throw new IllegalStateException("Duplicate special header found " + header.key());
    }
    map.put(header.key(), header.value());
  }
  return map;
}
 

@Test
public void testConsumeWithHeader(TestContext ctx) {
  MockConsumer<String, String> mock = new MockConsumer<>(OffsetResetStrategy.EARLIEST);
  KafkaReadStream<String, String> consumer = createConsumer(vertx, mock);
  Async doneLatch = ctx.async();
  consumer.handler(record -> {
    ctx.assertEquals("the_topic", record.topic());
    ctx.assertEquals(0, record.partition());
    ctx.assertEquals("abc", record.key());
    ctx.assertEquals("def", record.value());
    Header[] headers = record.headers().toArray();
    ctx.assertEquals(1, headers.length);
    Header header = headers[0];
    ctx.assertEquals("header_key", header.key());
    ctx.assertEquals("header_value", new String(header.value()));
    consumer.close(v -> doneLatch.complete());
  });
  consumer.subscribe(Collections.singleton("the_topic"), v -> {
    mock.schedulePollTask(() -> {
      mock.rebalance(Collections.singletonList(new TopicPartition("the_topic", 0)));
      mock.addRecord(new ConsumerRecord<>("the_topic", 0, 0L, 0L, TimestampType.NO_TIMESTAMP_TYPE, 0L, 0, 0, "abc", "def",
        new RecordHeaders(Collections.singletonList(new RecordHeader("header_key", "header_value".getBytes())))));
      mock.seek(new TopicPartition("the_topic", 0), 0L);
    });
  });
}
 

@Test
public void testStreamWithHeader(TestContext ctx) {
  int numMessages = 1000;
  String topicName = "testStreamWithHeader";
  Properties config = setupConsumeWithHeaders(ctx, numMessages, topicName);
  consumer = createConsumer(vertx, config);
  Async done = ctx.async();
  AtomicInteger count = new AtomicInteger(numMessages);
  AtomicInteger headerIndex = new AtomicInteger();
  consumer.exceptionHandler(ctx::fail);
  consumer.handler(rec -> {
    Header[] headers = rec.headers().toArray();
    ctx.assertEquals(1, headers.length);
    Header header = headers[0];
    ctx.assertEquals("header_key" + headerIndex.get(), header.key());
    ctx.assertEquals("header_value" + headerIndex.getAndIncrement(), new String(header.value()));
    if (count.decrementAndGet() == 0) {
      done.complete();
    }
  });
  consumer.subscribe(Collections.singleton(topicName));
}
 

/**
 * Creates a new outgoing Kafka Message with a header added to the header list.
 *
 * @param key the header key
 * @param content the header key, must not be {@code null}
 * @return the updated Kafka Message.
 */
public OutgoingKafkaRecord<K, T> withHeader(String key, byte[] content) {
    Headers headers = getHeaders();
    Headers copy = new RecordHeaders(headers);
    copy.add(new Header() {
        @Override
        public String key() {
            return key;
        }

        @Override
        public byte[] value() {
            return content;
        }
    });
    return new OutgoingKafkaRecord<>(getTopic(), getKey(), getPayload(), getTimestamp(), getPartition(),
            copy, getAck(), getNack(), getMetadata());
}
 

/**
 * Creates a new outgoing Kafka Message with a header added to the header list.
 *
 * @param key the header key
 * @param content the header key, must not be {@code null}
 * @return the updated Kafka Message.
 */
public OutgoingKafkaRecord<K, T> withHeader(String key, String content) {
    Headers headers = getHeaders();
    Headers copy = new RecordHeaders(headers);
    copy.add(new Header() {
        @Override
        public String key() {
            return key;
        }

        @Override
        public byte[] value() {
            return content.getBytes();
        }
    });
    return new OutgoingKafkaRecord<>(getTopic(), getKey(), getPayload(), getTimestamp(), getPartition(),
            copy, getAck(), getNack(), getMetadata());
}
 

/**
 * Creates a new outgoing Kafka Message with a header added to the header list.
 *
 * @param key the header key
 * @param content the header key, must not be {@code null}
 * @param enc the encoding, must not be {@code null}
 * @return the updated Kafka Message.
 */
public OutgoingKafkaRecord<K, T> withHeader(String key, String content, Charset enc) {
    Headers headers = getHeaders();
    Headers copy = new RecordHeaders(headers);
    copy.add(new Header() {
        @Override
        public String key() {
            return key;
        }

        @Override
        public byte[] value() {
            return content.getBytes(enc);
        }
    });
    return new OutgoingKafkaRecord<>(getTopic(), getKey(), getPayload(), getTimestamp(), getPartition(),
            copy, getAck(), getNack(), getMetadata());
}
 

private ProducerRecord<?, ?> getProducerRecord(Message<?> message, OutgoingKafkaRecordMetadata<?> om,
        String actualTopic) {
    int actualPartition = om == null || om.getPartition() <= -1 ? this.partition : om.getPartition();
    Object actualKey = om == null || om.getKey() == null ? key : om.getKey();

    long actualTimestamp;
    if ((om == null) || (om.getKey() == null)) {
        actualTimestamp = -1;
    } else {
        actualTimestamp = (om.getTimestamp() != null) ? om.getTimestamp().toEpochMilli() : -1;
    }

    Iterable<Header> kafkaHeaders = om == null || om.getHeaders() == null ? Collections.emptyList() : om.getHeaders();
    return new ProducerRecord<>(
            actualTopic,
            actualPartition == -1 ? null : actualPartition,
            actualTimestamp == -1L ? null : actualTimestamp,
            actualKey,
            message.getPayload(),
            kafkaHeaders);
}
 

private Envelope toEnvelope(String topic, ConsumerRecord<ByteBuffer, ByteBuffer> record) {
    var headers = new HashMap<String, String>();
    for (Header header : record.headers()) {
        headers.put(header.key(), new String(header.value()));
    }

    return toEnvelope(
            topic,
            record.key(),
            record.value(),
            headers
    );
}
 

public static EventOwnerHeader deserialize(final ConsumerRecord<byte[], byte[]> record) {
    final Header nameHeader = record.headers().lastHeader(EventOwnerHeader.AUTH_PARAM_NAME);
    if (null == nameHeader) {
        return null;
    }
    final Header valueHeader = record.headers().lastHeader(EventOwnerHeader.AUTH_PARAM_VALUE);
    if (valueHeader == null) {
        return null;
    }

    return new EventOwnerHeader(
            new String(nameHeader.value(), Charsets.UTF_8),
            new String(valueHeader.value(), Charsets.UTF_8));
}
 

private static ProducerMessage.Envelope<String, String, PassThrough> mapExternalMessageToKafkaMessage(
        final KafkaPublishTarget publishTarget, final ExternalMessage externalMessage,
        final PassThrough passThrough) {

    final String payload = mapExternalMessagePayload(externalMessage);
    final Iterable<Header> headers = mapExternalMessageHeaders(externalMessage);

    final ProducerRecord<String, String> record =
            new ProducerRecord<>(publishTarget.getTopic(),
                    publishTarget.getPartition().orElse(null),
                    publishTarget.getKey().orElse(null),
                    payload, headers);
    return ProducerMessage.single(record, passThrough);
}
 

void clearTraceIdHeaders(Headers headers) {
  // Headers::remove creates and consumes an iterator each time. This does one loop instead.
  for (Iterator<Header> i = headers.iterator(); i.hasNext(); ) {
    Header next = i.next();
    if (traceIdHeaders.contains(next.key())) i.remove();
  }
}
 

public HeadersMapExtractAdapter(Headers headers) {
  for (Header header : headers) {
    byte[] headerValue = header.value();
    map.put(header.key(),
        headerValue == null ? null : new String(headerValue, StandardCharsets.UTF_8));
  }
}
 

static Map<String, String> lastHeaders(MockProducer<Object, String> mockProducer) {
  Map<String, String> headers = new LinkedHashMap<>();
  List<ProducerRecord<Object, String>> history = mockProducer.history();
  ProducerRecord<Object, String> lastRecord = history.get(history.size() - 1);
  for (Header header : lastRecord.headers()) {
    headers.put(header.key(), new String(header.value(), StandardCharsets.UTF_8));
  }
  return headers;
}
 

private void reconsumeLater(ConsumerRecord<String, byte[]> consumeRecord) throws InterruptedException, ExecutionException {

		// add all header to headList except RETRY_COUNT
		Headers headers = consumeRecord.headers();
		List<Header> headerList = new ArrayList<Header>(8);
		Iterator<Header> iterator = headers.iterator();
		Integer retryCount = -1;
		boolean hasOrignalHeader = false;
		while (iterator.hasNext()) {
			Header next = iterator.next();
			if (next.key().equals(RETRY_COUNT_KEY)) {
				retryCount = serializer.deserialize(next.value());
				continue;
			}
			
			if(next.key().equals(ORGINAL_TOPIC)){
				hasOrignalHeader = true;
			}
			headerList.add(next);
		}
		
		// add RETRY_COUNT to header
		retryCount++;
		headerList.add(new RecordHeader(RETRY_COUNT_KEY, serializer.serialization(retryCount)));
		
		if(!hasOrignalHeader){
			headerList.add(new RecordHeader(ORGINAL_TOPIC, serializer.serialization(consumeRecord.topic())));
		}

		// send message to corresponding queue according to retry times
		String retryTopic = calcRetryTopic(consumeRecord.topic(), retryCount);
		
		ProducerRecord<String, byte[]> record = new ProducerRecord<>(retryTopic,
				consumeRecord.partition() % retryQueuePartitionCount.get(retryTopic), null, consumeRecord.key(),
				consumeRecord.value(), headerList);
		Future<RecordMetadata> publishKafkaMessage = retryQueueMsgProducer.publishKafkaMessage(record);
		publishKafkaMessage.get();
	}
 

/**
 * Extract the keys as a {@link Set} from the given {@code headers}.
 *
 * @param headers the Kafka {@code headers} to extract a key {@link Set} from
 * @return the keys of the given {@code headers}
 */
public static Set<String> keys(Headers headers) {
    notNull(headers, () -> "Headers may not be null");

    return StreamSupport.stream(headers.spliterator(), false)
                        .map(Header::key)
                        .collect(Collectors.toSet());
}
 

private Iterable<KV<String, byte[]>> toIterable(KafkaRecord record) {
  if (!ConsumerSpEL.hasHeaders) {
    return Collections.emptyList();
  }

  List<KV<String, byte[]>> vals = new ArrayList<>();
  for (Header header : record.getHeaders()) {
    vals.add(KV.of(header.key(), header.value()));
  }
  return vals;
}
 

/**
 * Gets the global id from the headers.  Returns null if not found.
 *
 * @param headers the headers
 */
protected Long getGlobalId(Headers headers) {
    Header header = headers.lastHeader(JsonSchemaSerDeConstants.HEADER_GLOBAL_ID);
    if (header == null) {
        return null;
    }
    return ByteBuffer.wrap(header.value()).getLong();
}
 

private de.otto.synapse.message.Header toHeader(final ConsumerRecord<String, String> record) {
    de.otto.synapse.message.Header.Builder builder = de.otto.synapse.message.Header.builder()
            .withShardPosition(fromPosition("" + record.partition(), "" + (record.offset())));
    record.headers().forEach(header -> {
        builder.withAttribute(header.key(), toString(header.value()));
    });
    return builder.build();
}