org.apache.kafka.connect.header.Header Java Examples

@Override
public void put(Collection<SinkRecord> sinkRecords) {
    for (SinkRecord record : sinkRecords) {
        TaskHelper.logRecordContent(LOG, record, config);
        Map<String, Object> headers = new HashMap<String, Object>();
        Exchange exchange = new DefaultExchange(producer.getCamelContext());
        headers.put(KAFKA_RECORD_KEY_HEADER, record.key());
        for (Iterator<Header> iterator = record.headers().iterator(); iterator.hasNext();) {
            Header header = (Header)iterator.next();
            if (header.key().startsWith(HEADER_CAMEL_PREFIX)) {
                addHeader(headers, header);
            } else if (header.key().startsWith(PROPERTY_CAMEL_PREFIX)) {
                addProperty(exchange, header);
            }
        }
        exchange.getMessage().setHeaders(headers);
        exchange.getMessage().setBody(record.value());

        LOG.debug("Sending exchange {} to {}", exchange.getExchangeId(), LOCAL_URL);
        producer.send(LOCAL_URL, exchange);

        if (exchange.isFailed()) {
            throw new ConnectException("Exchange delivery has failed!", exchange.getException());
        }
    }
}
 

private Iterable<? extends Header> getRecordHeaders(SinkRecord record) {
  ConnectHeaders headers = new ConnectHeaders();
  if(record.headers() != null) {
    int headerCount = 0;
    for (Header header : record.headers()) {
      if (header.key().getBytes().length < 257 &&
          String.valueOf(header.value()).getBytes().length < 1025) {
        headers.add(header);
        headerCount++;
      }
      if (headerCount > 100) {
        break;
      }
    }
  }
  return headers;
}
 

@TestFactory
public Stream<DynamicTest> headerValueDifferent() {
  return genericTests.entrySet().stream()
      .map(e -> dynamicTest(SchemaKey.of(e.getKey()).toString(), () -> {
        assertThrows(AssertionFailedError.class, () -> {
          Header expected = mock(Header.class);
          when(expected.schema()).thenReturn(e.getKey());
          when(expected.value()).thenReturn(e.getValue());
          when(expected.key()).thenReturn("key");
          Header actual = mock(Header.class);
          when(actual.schema()).thenReturn(e.getKey());
          when(actual.value()).thenReturn("adisfnbasd");
          when(actual.key()).thenReturn("key");
          assertHeader(expected, actual);
        });
      }));
}
 

private void addHeader(Map<String, Object> map, Header singleHeader) {
    String camelHeaderKey = StringUtils.removeStart(singleHeader.key(), HEADER_CAMEL_PREFIX);
    Schema schema = singleHeader.schema();
    if (schema.type().getName().equals(Schema.STRING_SCHEMA.type().getName())) {
        map.put(camelHeaderKey, (String)singleHeader.value());
    } else if (schema.type().getName().equalsIgnoreCase(Schema.BOOLEAN_SCHEMA.type().getName())) {
        map.put(camelHeaderKey, (Boolean)singleHeader.value());
    } else if (schema.type().getName().equalsIgnoreCase(Schema.INT32_SCHEMA.type().getName())) {
        map.put(camelHeaderKey, singleHeader.value());
    } else if (schema.type().getName().equalsIgnoreCase(Schema.BYTES_SCHEMA.type().getName())) {
        map.put(camelHeaderKey, (byte[])singleHeader.value());
    } else if (schema.type().getName().equalsIgnoreCase(Schema.FLOAT32_SCHEMA.type().getName())) {
        map.put(camelHeaderKey, (float)singleHeader.value());
    } else if (schema.type().getName().equalsIgnoreCase(Schema.FLOAT64_SCHEMA.type().getName())) {
        map.put(camelHeaderKey, (double)singleHeader.value());
    } else if (schema.type().getName().equalsIgnoreCase(Schema.INT16_SCHEMA.type().getName())) {
        map.put(camelHeaderKey, (short)singleHeader.value());
    } else if (schema.type().getName().equalsIgnoreCase(Schema.INT64_SCHEMA.type().getName())) {
        map.put(camelHeaderKey, (long)singleHeader.value());
    } else if (schema.type().getName().equalsIgnoreCase(Schema.INT8_SCHEMA.type().getName())) {
        map.put(camelHeaderKey, (byte)singleHeader.value());
    } else if (schema.type().getName().equalsIgnoreCase(SchemaBuilder.map(Schema.STRING_SCHEMA, Schema.STRING_SCHEMA).type().getName())) {
        map.put(camelHeaderKey, (Map<?, ?>)singleHeader.value());
    } else if (schema.type().getName().equalsIgnoreCase(SchemaBuilder.array(Schema.STRING_SCHEMA).type().getName())) {
        map.put(camelHeaderKey, (List<?>)singleHeader.value());
    }
}
 

@Override
public void serialize(Header header, JsonGenerator jsonGenerator, SerializerProvider serializerProvider) throws IOException, JsonProcessingException {
  KeyValue storage = new KeyValue();
  storage.name = header.key();
  storage.schema = header.schema();
  storage.value(header.value());
  jsonGenerator.writeObject(storage);
}
 

@TestFactory
public Stream<DynamicTest> headerActualNull() {
  return genericTests.entrySet().stream()
      .map(e -> dynamicTest(SchemaKey.of(e.getKey()).toString(), () -> {
        assertThrows(AssertionFailedError.class, () -> {
          Header expected = mock(Header.class);
          when(expected.schema()).thenReturn(e.getKey());
          when(expected.value()).thenReturn(e.getValue());
          when(expected.key()).thenReturn("key");
          assertHeader(expected, null);
        });
      }));
}
 

@TestFactory
public Stream<DynamicTest> headerExpectedNull() {
  return genericTests.entrySet().stream()
      .map(e -> dynamicTest(SchemaKey.of(e.getKey()).toString(), () -> {
        assertThrows(AssertionFailedError.class, () -> {
          Header expected = mock(Header.class);
          when(expected.schema()).thenReturn(e.getKey());
          when(expected.value()).thenReturn(e.getValue());
          when(expected.key()).thenReturn("key");
          assertHeader(null, expected);
        });
      }));
}
 

@TestFactory
public Stream<DynamicTest> headerEquals() {
  return genericTests.entrySet().stream()
      .map(e -> dynamicTest(SchemaKey.of(e.getKey()).toString(), () -> {
        Header expected = mock(Header.class);
        when(expected.schema()).thenReturn(e.getKey());
        when(expected.value()).thenReturn(e.getValue());
        when(expected.key()).thenReturn("key");
        assertHeader(expected, expected);
      }));
}
 

public static void assertHeader(Header expected, Header actual, String message) {
  final String prefix = null != message ? message + ": " : "";
  if (null == expected) {
    assertNull(actual, prefix + "actual should be null.");
    return;
  }
  assertNotNull(actual, prefix + "actual should not be null.");
  assertEquals(expected.key(), expected.key(), prefix + "key() does not match.");
  assertValue(expected.value(), actual.value(), prefix + "value() does not match.");
  assertSchema(expected.schema(), actual.schema(), prefix + "schema() does not match.");
}
 

private void addProperty(Exchange exchange, Header singleHeader) {
    String camelPropertyKey = StringUtils.removeStart(singleHeader.key(), PROPERTY_CAMEL_PREFIX);
    Schema schema = singleHeader.schema();
    if (schema.type().getName().equals(Schema.STRING_SCHEMA.type().getName())) {
        exchange.getProperties().put(camelPropertyKey, (String)singleHeader.value());
    } else if (schema.type().getName().equalsIgnoreCase(Schema.BOOLEAN_SCHEMA.type().getName())) {
        exchange.getProperties().put(camelPropertyKey, (Boolean)singleHeader.value());
    } else if (schema.type().getName().equalsIgnoreCase(Schema.INT32_SCHEMA.type().getName())) {
        exchange.getProperties().put(camelPropertyKey, singleHeader.value());
    } else if (schema.type().getName().equalsIgnoreCase(Schema.BYTES_SCHEMA.type().getName())) {
        exchange.getProperties().put(camelPropertyKey, (byte[])singleHeader.value());
    } else if (schema.type().getName().equalsIgnoreCase(Schema.FLOAT32_SCHEMA.type().getName())) {
        exchange.getProperties().put(camelPropertyKey, (float)singleHeader.value());
    } else if (schema.type().getName().equalsIgnoreCase(Schema.FLOAT64_SCHEMA.type().getName())) {
        exchange.getProperties().put(camelPropertyKey, (double)singleHeader.value());
    } else if (schema.type().getName().equalsIgnoreCase(Schema.INT16_SCHEMA.type().getName())) {
        exchange.getProperties().put(camelPropertyKey, (short)singleHeader.value());
    } else if (schema.type().getName().equalsIgnoreCase(Schema.INT64_SCHEMA.type().getName())) {
        exchange.getProperties().put(camelPropertyKey, (long)singleHeader.value());
    } else if (schema.type().getName().equalsIgnoreCase(Schema.INT8_SCHEMA.type().getName())) {
        exchange.getProperties().put(camelPropertyKey, (byte)singleHeader.value());
    } else if (schema.type().getName().equalsIgnoreCase(SchemaBuilder.map(Schema.STRING_SCHEMA, Schema.STRING_SCHEMA).type().getName())) {
        exchange.getProperties().put(camelPropertyKey, (Map<?, ?>)singleHeader.value());
    } else if (schema.type().getName().equalsIgnoreCase(SchemaBuilder.array(Schema.STRING_SCHEMA).type().getName())) {
        exchange.getProperties().put(camelPropertyKey, (List<?>)singleHeader.value());
    }
}
 

static JsonNode parseHeaders(Headers headers)
{
  ObjectNode result = MAPPER.createObjectNode();
  for (Header header: headers)
  {
    result.set(header.key(), convertToJson(header.schema(), header.value()));
  }
  return result;
}
 

@Test
public void testSourceRecordsWorksWithHeaders() {
  final String topic = "topica";
  final int partition = 0;
  final int offset = 123;
  final long timestamp = 314159;

  Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> records = new HashMap<>();
  Headers headers = new RecordHeaders();
  headers.add("h1", "v1".getBytes(StandardCharsets.UTF_8));
  headers.add("h2", "v2".getBytes(StandardCharsets.UTF_8));
  records.put(
      new TopicPartition(topic, partition),
      Collections.singletonList(newConsumerRecord(topic, partition, offset, timestamp, headers)));
  ConsumerRecords<byte[], byte[]> pollResult = new ConsumerRecords<>(records);

  List<SourceRecord> result = mirusSourceTask.sourceRecords(pollResult);

  assertThat(
      StreamSupport.stream(result.get(0).headers().spliterator(), false)
          .map(Header::key)
          .collect(Collectors.toList()),
      hasItems("h1", "h2"));
  assertThat(
      StreamSupport.stream(result.get(0).headers().spliterator(), false)
          .map(Header::value)
          .collect(Collectors.toList()),
      hasItems("v1".getBytes(StandardCharsets.UTF_8), "v2".getBytes(StandardCharsets.UTF_8)));
}
 

public void convert(ConnectRecord record, Struct struct) {
  final Header header = record.headers().lastWithName(this.header);
  Object fieldValue;
  if (null != header) {
    fieldValue = convert(header);
  } else {
    fieldValue = null;
  }

  struct.put(this.field, fieldValue);
}
 

public static void assertHeader(Header expected, Header actual) {
  assertHeader(expected, actual, null);
}
 

@Override
public Header rename(String s) {
  return new HeaderImpl(this.key, this.schema, this.value);
}
 

@Override
public Header with(Schema schema, Object value) {
  return new HeaderImpl(this.key, schema, value);
}
 

@Override
public Header deserialize(JsonParser jsonParser, DeserializationContext deserializationContext) throws IOException, JsonProcessingException {
  KeyValue storage = jsonParser.readValueAs(KeyValue.class);
  return new HeaderImpl(storage.name, storage.schema, storage.value());
}
 

public HeaderSerializationModule() {
  super();
  addSerializer(Header.class, new Serializer());
  addDeserializer(Header.class, new Deserializer());
}
 

@Test
public void testSourceRecordsWorksWithHeadersWithHeaderConverter() {
  final String topic = "topica";
  final int partition = 0;
  final int offset = 123;
  final long timestamp = 314159;

  Map<String, String> properties = mockTaskProperties();
  properties.put(
      SourceConfigDefinition.SOURCE_HEADER_CONVERTER.getKey(),
      "org.apache.kafka.connect.json.JsonConverter");

  mirusSourceTask.start(properties);

  Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> records = new HashMap<>();
  Headers headers = new RecordHeaders();
  headers.add(
      "h1",
      "{\"schema\": {\"type\": \"struct\",\"fields\": [{\"type\": \"string\",\"optional\": true,\"field\": \"version\"}],\"optional\": false},\"payload\": {\"version\": \"v1\"}}"
          .getBytes(StandardCharsets.UTF_8));
  headers.add(
      "h2",
      "{\"schema\": {\"type\": \"struct\",\"fields\": [{\"type\": \"string\",\"optional\": true,\"field\": \"version\"}],\"optional\": false},\"payload\": {\"version\": \"v2\"}}"
          .getBytes(StandardCharsets.UTF_8));
  records.put(
      new TopicPartition(topic, partition),
      Collections.singletonList(newConsumerRecord(topic, partition, offset, timestamp, headers)));
  ConsumerRecords<byte[], byte[]> pollResult = new ConsumerRecords<>(records);

  List<SourceRecord> result = mirusSourceTask.sourceRecords(pollResult);

  Iterator<Header> connectHeaders = result.get(0).headers().iterator();
  Header header1 = connectHeaders.next();
  assertThat(header1.key(), is("h1"));
  assertThat(header1.value(), instanceOf(Struct.class));
  Struct header1Value = (Struct) header1.value();
  assertThat(header1Value.getString("version"), is("v1"));

  Header header2 = connectHeaders.next();
  assertThat(header2.key(), is("h2"));
  assertThat(header2.value(), instanceOf(Struct.class));
  Struct header2Value = (Struct) header2.value();
  assertThat(header2Value.getString("version"), is("v2"));
}
 

@Override
Object convert(Header header) {
  return Values.convertToString(header.schema(), header.value());
}
 

@Override
Object convert(Header header) {
  return Values.convertToBoolean(header.schema(), header.value());
}
 

@Override
Object convert(Header header) {
  return Values.convertToDate(header.schema(), header.value());
}
 

@Override
Object convert(Header header) {
  return Values.convertToTime(header.schema(), header.value());
}
 

@Override
Object convert(Header header) {
  return Values.convertToTimestamp(header.schema(), header.value());
}
 

@Override
Object convert(Header header) {
  return Values.convertToDecimal(header.schema(), header.value(), scale);
}
 

@Override
Object convert(Header header) {
  return Values.convertToLong(header.schema(), header.value());
}
 

@Override
Object convert(Header header) {
  return Values.convertToInteger(header.schema(), header.value());
}
 

@Override
Object convert(Header header) {
  return Values.convertToShort(header.schema(), header.value());
}
 

@Override
Object convert(Header header) {
  return Values.convertToByte(header.schema(), header.value());
}
 

@Override
Object convert(Header header) {
  return Values.convertToDouble(header.schema(), header.value());
}