org.apache.kafka.connect.connector.ConnectRecord Java Examples

@Override
public void put(Collection<SinkRecord> records) {
    if (records.isEmpty()) {
        return;
    }
    try {
        List<Search.Artifact> artifacts = records.stream()
                                                 .map(ConnectRecord::value)
                                                 .map(Search.Artifact.class::cast)
                                                 .collect(Collectors.toList());

        log.info("Indexing artifacts ({}) ... ", artifacts.size());
        client.index(artifacts);
    } catch (Exception e) {
        e.printStackTrace();
        throw new IllegalStateException(e);
    }
}
 

void assertSourceRecord(final Map<String, ?> expected, final ConnectRecord record, final String topic) throws JsonProcessingException {
  assertNotNull(record, "record should not be null.");
  assertNotNull(record.value(), "record.value() should not be null.");
  assertEquals(topic, record.topic(), "topic does not match.");
  assertTrue(record.key() instanceof Struct, "record.key() should be a struct");
  assertTrue(record.value() instanceof Struct, "record.value() should be a struct");

  Struct keyStruct = (Struct) record.key();
  keyStruct.validate();

  Struct valueStruct = (Struct) record.value();
  valueStruct.validate();

  for (Map.Entry<String, ?> entry : expected.entrySet()) {
    Object structValue = valueStruct.get(entry.getKey());

    if (entry.getValue() instanceof Map) {
      String text = ObjectMapperFactory.INSTANCE.writeValueAsString(entry.getValue());
      String structText = (String) structValue;
      assertEquals(text, structText, entry.getKey() + " should match.");
    } else {
      assertEquals(entry.getValue(), structValue, entry.getKey() + " should match.");
    }
  }
}
 

public static void assertRecord(ConnectRecord expected, ConnectRecord actual, String message) {
  final String prefix = null != message ? message + ": " : "";
  if (null == expected) {
    assertNull(actual, prefix + "actual should be null.");
    return;
  }

  assertNotNull(actual, prefix + "actual should not be null.");
  assertEquals(expected.kafkaPartition(), actual.kafkaPartition(), prefix + "kafkaPartition() does not match.");
  assertEquals(expected.topic(), actual.topic(), prefix + "topic() does not match.");
  assertEquals(expected.timestamp(), actual.timestamp(), prefix + "timestamp() does not match.");
  assertSchema(expected.keySchema(), actual.keySchema(), prefix + "keySchema() does not match");
  assertValue(expected.key(), actual.key(), prefix + "key() does not match.");
  assertSchema(expected.valueSchema(), actual.valueSchema(), prefix + "valueSchema() does not match");
  assertValue(expected.value(), actual.value(), prefix + "value() does not match.");
}
 

public static void assertRecord(ConnectRecord expected, ConnectRecord actual, String message) {
  final String prefix = null != message ? message + ": " : "";
  if (null == expected) {
    assertNull(actual, prefix + "actual should be null.");
    return;
  }

  assertNotNull(actual, prefix + "actual should not be null.");
  assertEquals(expected.kafkaPartition(), actual.kafkaPartition(), prefix + "kafkaPartition() does not match.");
  assertEquals(expected.topic(), actual.topic(), prefix + "topic() does not match.");
  assertEquals(expected.timestamp(), actual.timestamp(), prefix + "timestamp() does not match.");
  assertSchema(expected.keySchema(), actual.keySchema(), prefix + "keySchema() does not match");
  assertValue(expected.key(), actual.key(), prefix + "key() does not match.");
  assertSchema(expected.valueSchema(), actual.valueSchema(), prefix + "valueSchema() does not match");
  assertValue(expected.value(), actual.value(), prefix + "value() does not match.");
  assertHeaders(expected.headers(), actual.headers(), prefix + "headers() does not match.");
}
 

public static <CFG extends AbstractConfig> void logRecordContent(Logger logger, ConnectRecord<?> record, CFG config) {
    if (logger != null && record != null && config != null) {
        // do not log record's content by default, as it may contain sensitive information
        LoggingLevel level = LoggingLevel.OFF;
        try {
            final String key = (record instanceof SourceRecord)
                ? CamelSourceConnectorConfig.CAMEL_SOURCE_CONTENT_LOG_LEVEL_CONF
                : CamelSinkConnectorConfig.CAMEL_SINK_CONTENT_LOG_LEVEL_CONF;
            level = LoggingLevel.valueOf(config.getString(key).toUpperCase());
        } catch (Exception e) {
            logger.warn("Invalid value for contentLogLevel property");
        }
        switch (level) {
            case TRACE:
                logger.trace(record.toString());
                break;
            case DEBUG:
                logger.debug(record.toString());
                break;
            case INFO:
                logger.info(record.toString());
                break;
            case WARN:
                logger.warn(record.toString());
                break;
            case ERROR:
                logger.error(record.toString());
                break;
            default:
                break;
        }
    }
}
 

private static JsonNode getValueAsJsonNode(ConnectRecord record) throws IOException {
  if (record.value() instanceof Map) {
    return objectMapper.convertValue(record.value(), JsonNode.class);
  }

  return objectMapper.readTree(getValueAsJsonBytes(record));
}
 

public void convert(ConnectRecord record, Struct struct) {
  final Header header = record.headers().lastWithName(this.header);
  Object fieldValue;
  if (null != header) {
    fieldValue = convert(header);
  } else {
    fieldValue = null;
  }

  struct.put(this.field, fieldValue);
}
 

public SchemaAndValue apply(ConnectRecord record, Struct input) {
  Struct result = new Struct(this.newSchema);
  for (Field field : input.schema().fields()) {
    String fieldName = field.name();
    Object fieldValue = input.get(field);
    result.put(fieldName, fieldValue);
  }
  for (ConversionHandler handler : this.conversionHandlers) {
    handler.convert(record, result);
  }
  return new SchemaAndValue(this.newSchema, result);
}
 

@Test
public void apply() throws IOException {
  final byte[] input = Files.toByteArray(new File("src/test/resources/com/github/jcustenborder/kafka/connect/transform/xml/books.xml"));
  final ConnectRecord inputRecord = new SinkRecord(
      "test",
      1,
      null,
      null,
      org.apache.kafka.connect.data.Schema.BYTES_SCHEMA,
      input,
      new Date().getTime()
  );

  ConnectRecord record = this.transform.apply(inputRecord);
}
 

@Test
public void testTombstoneRecord() {
    configure(false);

    ConnectRecord record = createRecord(null, null, Schema.OPTIONAL_BYTES_SCHEMA, null);

    log.info("applying transformation");
    ConnectRecord appliedRecord = assertDoesNotThrow(() -> smt.apply(record));

    assertEquals(record.valueSchema(), appliedRecord.valueSchema(), "value schema unchanged");
    assertNull(appliedRecord.value());
}
 

@Test
public void testValueSchemaLookupFailure() {
    configure(false);

    byte[] b = ByteBuffer.allocate(6).array();
    ConnectRecord record = createRecord(null, null, null, b);

    // tries to lookup schema id 0, but that isn't a valid id
    assertThrows(ConnectException.class, () -> smt.apply(record));
}
 

@Test
public void testKeySchemaLookupFailure() {
    configure(true);

    byte[] b = ByteBuffer.allocate(6).array();
    ConnectRecord record = createRecord(null, b, null, null);

    // tries to lookup schema id 0, but that isn't a valid id
    assertThrows(ConnectException.class, () -> smt.apply(record));
}
 

@Test
public void applySchemalessValueBytesTooShort() {
    configure(false);

    // allocate enough space for the magic-byte
    byte[] b = ByteBuffer.allocate(1).array();
    ConnectRecord record = createRecord(null, null, null, b);

    // The value payload is not long enough for schema registry wire-format
    assertThrows(SerializationException.class, () -> smt.apply(record));
}
 

@Test
public void applySchemalessKeyBytesTooShort() {
    configure(true);

    // allocate enough space for the magic-byte
    byte[] b = ByteBuffer.allocate(1).array();
    ConnectRecord record = createRecord(null, b, null, null);

    // The key payload is not long enough for schema registry wire-format
    assertThrows(SerializationException.class, () -> smt.apply(record));
}
 

@Test
public void applyValueSchemaNotBytes() {
    configure(false);

    ConnectRecord record = createRecord(null, null, null, null);

    // The value schema is not a byte[]
    assertThrows(ConnectException.class, () -> smt.apply(record));
}
 

@Test
public void applyKeySchemaNotBytes() {
    configure(true);

    ConnectRecord record = createRecord(null, null, null, null);

    // The key schema is not a byte[]
    assertThrows(ConnectException.class, () -> smt.apply(record));
}
 

private void passSimpleMessage() throws IOException {
    // Create key/value schemas for source registry
    log.info("Registering key/value string schemas in source registry");
    final int sourceKeyId = sourceSchemaRegistry.registerSchema(TOPIC, true, STRING_SCHEMA);
    final int sourceValId = sourceSchemaRegistry.registerSchema(TOPIC, false, STRING_SCHEMA);

    final ByteArrayOutputStream keyOut =
        encodeAvroObject(STRING_SCHEMA, sourceKeyId, HELLO_WORLD_VALUE);
    final ByteArrayOutputStream valOut =
        encodeAvroObject(STRING_SCHEMA, sourceValId, HELLO_WORLD_VALUE);
    final ConnectRecord record =
        createRecord(keyOut.toByteArray(), valOut.toByteArray());

    smt.apply(record);
}
 

public static void assertRecord(ConnectRecord expected, ConnectRecord actual) {
  assertRecord(expected, actual, null);
}
 

ConnectTemplateHashModel(ConnectRecord connectRecord, Struct struct) {
  this.connectRecord = connectRecord;
  this.struct = struct;
}
 

public String execute(String templateName, ConnectRecord record, Struct struct, Map<String, ?> additionalValues) {
  ConnectTemplateHashModel connectTemplateHashModel = new ConnectTemplateHashModel(record, struct);
  return executeInternal(templateName, connectTemplateHashModel);
}
 

public static void assertRecord(ConnectRecord expected, ConnectRecord actual) {
  assertRecord(expected, actual, null);
}
 

private ConnectRecord createRecord(byte[] key, byte[] value) {
    return createRecord(Schema.OPTIONAL_BYTES_SCHEMA, key, Schema.OPTIONAL_BYTES_SCHEMA, value);
}
 

@SuppressWarnings("unchecked")
private static Map<String, Object> getValueAsMap(ConnectRecord record) throws IOException {
  return record.value() instanceof Map
      ? (Map) record.value()
      : objectMapper.convertValue(getValueAsJsonNode(record), Map.class);
}
 

private ConnectRecord createRecord(Schema keySchema, Object key, Schema valueSchema, Object value) {
    // partition and offset aren't needed
    return new SourceRecord(null, null, TOPIC, keySchema, key, valueSchema, value);
}
 

private static byte[] getValueAsJsonBytes(ConnectRecord record) {
  return jsonConverter.fromConnectData(record.topic(), record.valueSchema(), record.value());
}