org.apache.kafka.connect.data.SchemaAndValue Java Examples

/**
 * cast bytes array to Json array
 *
 * @param s     topic name. unused
 * @param bytes input bytes array, only support single json record now
 * @return JSON array
 */
@Override
public SchemaAndValue toConnectData(final String s, final byte[] bytes)
{
  if(bytes == null)
  {
    return new SchemaAndValue(new SnowflakeJsonSchema(), new SnowflakeRecordContent());
  }
  try
  {
    //always return an array of JsonNode because AVRO record may contains
    // multiple records
    return new SchemaAndValue(new SnowflakeJsonSchema(),
      new SnowflakeRecordContent(mapper.readTree(bytes)));
  } catch (Exception ex)
  {
    LOGGER.error(Logging.logMessage("Failed to parse JSON record\n" + ex.toString()));
    return new SchemaAndValue(new SnowflakeJsonSchema(),
      new SnowflakeRecordContent(bytes));
  }
}
 

@Override
protected List<SourceRecord> process() {
  int recordCount = 0;
  List<SourceRecord> records = new ArrayList<>(this.config.batchSize);

  LogEntry entry;
  try {
    while (null != (entry = next()) && recordCount < this.config.batchSize) {
      log.trace("process() - Processing LogEntry: {}", entry);
      SchemaAndValue value = conversion.convert(entry);
      SourceRecord record = record(SchemaAndValue.NULL, value, null);
      records.add(record);
      recordCount++;
    }
  } catch (IOException ex) {
    throw new ConnectException(ex);
  }
  return records;
}
 

@Override
protected List<SourceRecord> process() throws IOException {
  int recordCount = 0;
  List<SourceRecord> records = new ArrayList<>(this.config.batchSize);
  while (recordCount < this.config.batchSize && this.nodeIterator.hasNext()) {
    JsonNode node = this.nodeIterator.next();
    String value = ObjectMapperFactory.INSTANCE.writeValueAsString(node);
    SourceRecord record = record(
        null,
        new SchemaAndValue(Schema.STRING_SCHEMA, value),
        null
    );
    records.add(record);
    recordCount++;
    recordOffset++;
  }
  return records;
}
 

private long processMap(SchemaAndValue schemaAndValue) {
  Preconditions.checkState(schemaAndValue.value() instanceof Map, "value must be a map.");
  final Map<String, Object> input = (Map<String, Object>) schemaAndValue.value();
  final Object inputValue = input.get(this.config.fieldName);
  final long result;

  if (inputValue instanceof Date) {
    final Date inputDate = (Date) inputValue;
    result = inputDate.getTime();
  } else if (inputValue instanceof Long) {
    result = (long) inputValue;
  } else if (null == inputValue) {
    throw new DataException(
        String.format("Field '%s' cannot be null.", this.config.fieldName)
    );
  } else {
    throw new DataException(
        String.format("Cannot convert %s to timestamp.", inputValue.getClass().getName())
    );
  }

  return result;
}
 

/**
 * Gets the value schema to use for the Kafka Connect SourceRecord.
 *
 * @param context            the JMS context to use for building messages
 * @param topic              the Kafka topic
 * @param messageBodyJms     whether to interpret MQ messages as JMS messages
 * @param message            the message
 *
 * @return the Kafka Connect SourceRecord's value
 *
 * @throws JMSException      Message could not be converted
 */
@Override public SchemaAndValue getValue(JMSContext context, String topic, boolean messageBodyJms, Message message) throws JMSException {
    byte[] payload;

    if (message instanceof BytesMessage) {
        payload = message.getBody(byte[].class);
    }
    else if (message instanceof TextMessage) {
        String s = message.getBody(String.class);
        payload = s.getBytes(UTF_8);
    }
    else {
        log.error("Unsupported JMS message type {}", message.getClass());
        throw new ConnectException("Unsupported JMS message type");
    }

    return converter.toConnectData(topic, payload);
}
 

private void insert(SnowflakeSinkService sink, int partition,
                               int numOfRecord)
{
  ExecutorService executorService = Executors.newSingleThreadExecutor();

  executorService.submit(
    () ->
    {
      for (int i = 0; i < numOfRecord; i++)
      {
        SnowflakeConverter converter = new SnowflakeJsonConverter();
        SchemaAndValue input = converter.toConnectData(topic, "{\"name\":\"test\"}".getBytes());

        sink.insert(
          new SinkRecord(topic, partition, Schema.STRING_SCHEMA
            , "test", input.schema(), input.value(), i));
      }
    }
  );
}
 

/** {@inheritDoc} */
@Override public SchemaAndValue toConnectData(String topic, byte[] bytes) {
    CacheEvent evt;

    try {
        evt = deserializer.deserialize(topic, bytes);
    }
    catch (SerializationException e) {
        throw new DataException("Failed to convert to Kafka Connect data due to a serialization error", e);
    }

    if (evt == null) {
        return SchemaAndValue.NULL;
    }
    return new SchemaAndValue(null, evt);
}
 

public static SinkRecord write(String topic, SchemaAndValue key, SchemaAndValue value) {
  Preconditions.checkNotNull(topic, "topic cannot be null");
  Preconditions.checkNotNull(key, "key cannot be null.");
  Preconditions.checkNotNull(key.value(), "key cannot be null.");
  Preconditions.checkNotNull(value, "value cannot be null.");
  Preconditions.checkNotNull(value.value(), "value cannot be null.");

  return new SinkRecord(
      topic,
      PARTITION,
      key.schema(),
      key.value(),
      value.schema(),
      value.value(),
      OFFSET,
      TIMESTAMP,
      TimestampType.CREATE_TIME
  );
}
 

SchemaAndValue schemaAndValue(Schema inputSchema, Object input) {
  final byte[] buffer = this.converter.fromConnectData("dummy", inputSchema, input);
  final Schema schema;
  final Object value;

  switch (this.config.outputSchema) {
    case STRING:
      value = new String(buffer, Charsets.UTF_8);
      schema = Schema.OPTIONAL_STRING_SCHEMA;
      break;
    case BYTES:
      value = buffer;
      schema = Schema.OPTIONAL_BYTES_SCHEMA;
      break;
    default:
      throw new UnsupportedOperationException(
          String.format(
              "Schema type (%s)'%s' is not supported.",
              ToJSONConfig.OUTPUT_SCHEMA_CONFIG,
              this.config.outputSchema
          )
      );
  }

  return new SchemaAndValue(schema, value);
}
 

@Override
protected List<SourceRecord> process() throws IOException {
  int recordCount = 0;
  List<SourceRecord> records = new ArrayList<>(this.config.batchSize);
  String line = null;
  while (recordCount < this.config.batchSize && null != (line = this.reader.readLine())) {
    SourceRecord record = record(
        null,
        new SchemaAndValue(Schema.STRING_SCHEMA, line),
        null
    );
    records.add(record);
    recordCount++;
  }
  return records;
}
 

@Before
public void setup() {
  layout = new StructuredLayout(converter, () -> builder);
  when(logEvent.getMessage()).thenReturn(log4jMessage);
  when(logEvent.getLevel()).thenReturn(LOG_LEVEL);
  when(logEvent.getLoggerName()).thenReturn(LOGGER_NAME);
  when(logEvent.getTimeMillis()).thenReturn(LOG_TIME_MS);
  when(converter.apply(any(Struct.class)))
      .thenReturn(SERIALIZED_MSG);
  schemaAndValue = new SchemaAndValue(schema, struct);
  when(logMessage.getMessage()).thenReturn(schemaAndValue);
  when(builder.withLevel(anyString())).thenReturn(builder);
  when(builder.withLoggerName(anyString())).thenReturn(builder);
  when(builder.withMessageSchemaAndValue(any(SchemaAndValue.class)))
      .thenReturn(builder);
  when(builder.withTimeMs(anyLong())).thenReturn(builder);
  when(struct.schema()).thenReturn(schema);
}
 

/**
 * Gets the value schema to use for the Kafka Connect SourceRecord.
 *
 * @param context            the JMS context to use for building messages
 * @param topic              the Kafka topic
 * @param messageBodyJms     whether to interpret MQ messages as JMS messages
 * @param message            the message
 *
 * @return the Kafka Connect SourceRecord's value
 *
 * @throws JMSException      Message could not be converted
 */
@Override public SchemaAndValue getValue(JMSContext context, String topic, boolean messageBodyJms, Message message) throws JMSException {
    Schema valueSchema = null;
    Object value = null;

    // Interpreting the body as a JMS message type, we can accept BytesMessage and TextMessage only.
    // We do not know the schema so do not specify one.
    if (messageBodyJms) {
        if (message instanceof BytesMessage) {
            log.debug("Bytes message with no schema");
            value = message.getBody(byte[].class);
        }
        else if (message instanceof TextMessage) {
            log.debug("Text message with no schema");
            value = message.getBody(String.class);
        }
        else {
            log.error("Unsupported JMS message type {}", message.getClass());
            throw new ConnectException("Unsupported JMS message type");
        }
    }
    else {
        // Not interpreting the body as a JMS message type, all messages come through as BytesMessage.
        // In this case, we specify the value schema as OPTIONAL_BYTES.
        log.debug("Bytes message with OPTIONAL_BYTES schema");
        valueSchema = Schema.OPTIONAL_BYTES_SCHEMA;
        value = message.getBody(byte[].class);
    }

    return new SchemaAndValue(valueSchema, value);
}
 

@Override
public R apply(R r) {
  final SchemaAndValue transformed = process(r, r.valueSchema(), r.value());

  return r.newRecord(
      r.topic(),
      r.kafkaPartition(),
      r.keySchema(),
      r.key(),
      transformed.schema(),
      transformed.value(),
      r.timestamp()
  );
}
 

@Override
protected List<SourceRecord> process() {
  List<SourceRecord> records = new ArrayList<>(this.config.batchSize);

  while (this.iterator.hasNext() && records.size() < this.config.batchSize) {
    JsonNode node = next();

    Struct valueStruct = new Struct(this.config.valueSchema);
    Struct keyStruct = new Struct(this.config.keySchema);
    log.trace("process() - input = {}", node);
    for (Field field : this.config.valueSchema.fields()) {
      JsonNode fieldNode = node.get(field.name());
      log.trace("process() - field: {} input = '{}'", field.name(), fieldNode);
      Object fieldValue;
      try {
        fieldValue = this.parser.parseJsonNode(field.schema(), fieldNode);
        log.trace("process() - field: {} output = '{}'", field.name(), fieldValue);
        valueStruct.put(field, fieldValue);

        Field keyField = this.config.keySchema.field(field.name());
        if (null != keyField) {
          log.trace("process() - Setting key field '{}' to '{}'", keyField.name(), fieldValue);
          keyStruct.put(keyField, fieldValue);
        }
      } catch (Exception ex) {
        String message = String.format("Exception thrown while parsing data for '%s'. linenumber=%s", field.name(), this.recordOffset());
        throw new DataException(message, ex);
      }
    }

    addRecord(
        records,
        new SchemaAndValue(keyStruct.schema(), keyStruct),
        new SchemaAndValue(valueStruct.schema(), valueStruct)
    );
  }

  return records;
}
 

protected long process(SchemaAndValue schemaAndValue) {
  final long result;
  if (schemaAndValue.value() instanceof Struct) {
    result = processStruct(schemaAndValue);
  } else if (schemaAndValue.value() instanceof Map) {
    result = processMap(schemaAndValue);
  } else {
    throw new UnsupportedOperationException();
  }
  return result;
}
 

@Override
public SchemaAndValue toConnectData(String topic, byte[] value) {
    if (value == null) return null;
    MeasurementV1 measurement = internalAvro.fromBytes(value);
    Struct result = new Struct(schema);
    result.put("timestamp", measurement.getTimestamp());
    result.put("name", measurement.getName());
    result.put("tags", measurement.getTags());
    result.put("fields", measurement.getFields());
    return new SchemaAndValue(schema, result);
}
 

public static SchemaAndValue getAvroWithoutRegistryKey() throws IOException
{
  SnowflakeAvroConverterWithoutSchemaRegistry avroConverterWithoutSchemaRegistry = new SnowflakeAvroConverterWithoutSchemaRegistry();

  URL resource = ConverterTest.class.getResource(TEST_KEY_FILE_NAME);
  byte[] value = Files.readAllBytes(Paths.get(resource.getFile()));

  return avroConverterWithoutSchemaRegistry.toConnectData(topic, value);
}
 

@Override
public R apply(R r) {
  final long timestamp = process(new SchemaAndValue(r.valueSchema(), r.value()));
  return r.newRecord(
      r.topic(),
      r.kafkaPartition(),
      r.keySchema(),
      r.key(),
      r.valueSchema(),
      r.value(),
      timestamp
  );
}
 

@Test
public void shouldCreateLoggerCorrectly() {
  // Given:
  when(innerFactory.apply("prefix.foo")).thenReturn(innerLogger);

  // When:
  final StructuredLogger logger = loggerFactory.getLogger("foo");
  logger.info(mock(SchemaAndValue.class));

  // Then:
  verify(innerFactory, times(1)).apply("prefix.foo");
  verify(innerLogger).info(any(String.class), any(SerializableSchemaAndValue.class));
}
 

@Override
public R apply(R r) {
  final SchemaAndValue transformed = process(r, r.keySchema(), r.key());

  return r.newRecord(
      r.topic(),
      r.kafkaPartition(),
      transformed.schema(),
      transformed.value(),
      r.valueSchema(),
      r.value(),
      r.timestamp()
  );
}
 

@Override
public R apply(R r) {
  final SchemaAndValue transformed = process(r, r.keySchema(), r.key());

  return r.newRecord(
      r.topic(),
      r.kafkaPartition(),
      transformed.schema(),
      transformed.value(),
      r.valueSchema(),
      r.value(),
      r.timestamp()
  );
}
 

@Override
protected SchemaAndValue processStruct(R record, Schema inputSchema, Struct input) {
  Date timestamp = new Date(this.time.milliseconds());

  Schema outputSchema = schemaCache.computeIfAbsent(inputSchema, schema -> {
    Collection<String> replaceFields = schema.fields().stream()
        .filter(f -> this.config.fields.contains(f.name()))
        .filter(f -> !isTimestampSchema(f.schema()))
        .map(Field::name)
        .collect(Collectors.toList());
    SchemaBuilder builder = SchemaBuilders.of(schema, replaceFields);
    this.config.fields.forEach(timestampField -> {
      Field existingField = builder.field(timestampField);
      if (null == existingField) {
        builder.field(timestampField, Timestamp.SCHEMA);
      }
    });
    return builder.build();
  });

  Struct output = new Struct(outputSchema);
  inputSchema.fields().stream()
      .filter(f -> !this.config.fields.contains(f.name()))
      .forEach(f -> output.put(f.name(), input.get(f.name())));
  this.config.fields.forEach(field -> output.put(field, timestamp));
  return new SchemaAndValue(outputSchema, output);
}
 

@Test
  public void foo() {
    ElfParser parser = mock(ElfParser.class);
    final Map<String, Class<?>> fieldTypes = ImmutableMap.of(
        "date", LocalDate.class,
        "time", LocalTime.class,
        "sc-bytes", Long.class,
        "sc-status", Integer.class
    );
    final Map<String, Object> fieldData = ImmutableMap.of(
        "date", LocalDate.of(2011, 3, 14),
        "time", LocalTime.of(12, 0, 0),
        "sc-bytes", 12341L,
        "sc-status", 200
    );
    when(parser.fieldTypes()).thenReturn(fieldTypes);

    SchemaConversionBuilder schemaGenerator = new SchemaConversionBuilder(parser);
    SchemaConversion conversion = schemaGenerator.build();
    assertNotNull(conversion, "conversion should not be null.");

    LogEntry entry = mock(LogEntry.class);
    when(entry.fieldTypes()).thenReturn(fieldTypes);
    when(entry.fieldData()).thenReturn(fieldData);

    SchemaAndValue actual = conversion.convert(entry);
    assertNotNull(actual, "actual should not be null");
//    assertNotNull(actual.getKey(), "actual.getKey() should not be null");
    assertNotNull(actual.schema(), "actual.getValue() should not be null");
    assertNotNull(actual.value(), "actual.getValue() should not be null");

//    actual.getValue()..validate();

//date time x-edge-location sc-bytes c-ip cs-method cs(Host) cs-uri-stem sc-status cs(Referer) cs(User-Agent) cs-uri-query cs(Cookie) x-edge-result-type x-edge-request-id x-host-header cs-protocol cs-bytes time-taken


  }
 

@Override
public R apply(R r) {
  final SchemaAndValue transformed = normalize(r, r.valueSchema(), r.value());

  return r.newRecord(
      r.topic(),
      r.kafkaPartition(),
      r.keySchema(),
      r.key(),
      transformed.schema(),
      transformed.value(),
      r.timestamp()
  );
}
 

@Override
public R apply(R r) {
  final SchemaAndValue transformed = process(r, r.keySchema(), r.key());

  return r.newRecord(
      r.topic(),
      r.kafkaPartition(),
      transformed.schema(),
      transformed.value(),
      r.valueSchema(),
      r.value(),
      r.timestamp()
  );
}
 

@Override
public R apply(R r) {
  final SchemaAndValue transformed = process(r, r.keySchema(), r.key());

  return r.newRecord(
      r.topic(),
      r.kafkaPartition(),
      transformed.schema(),
      transformed.value(),
      r.valueSchema(),
      r.value(),
      r.timestamp()
  );
}
 

@Override
public R apply(R r) {
  final SchemaAndValue transformed = process(r, r.valueSchema(), r.value());

  return r.newRecord(
      r.topic(),
      r.kafkaPartition(),
      r.keySchema(),
      r.key(),
      transformed.schema(),
      transformed.value(),
      r.timestamp()
  );
}
 

public SchemaAndValue toConnectData(String topic, byte[] value) {
  try {
    return new SchemaAndValue(Schema.OPTIONAL_BYTES_SCHEMA, this.deserializer.deserialize(topic, value));
  } catch (SerializationException var4) {
    throw new DataException("Failed to deserialize byte: ", var4);
  }
}
 

@Override
public R apply(R r) {
  final SchemaAndValue transformed = process(r, r.keySchema(), r.key());

  return r.newRecord(
      r.topic(),
      r.kafkaPartition(),
      transformed.schema(),
      transformed.value(),
      r.valueSchema(),
      r.value(),
      r.timestamp()
  );
}
 

@Test
public void shouldSerializeMessageToJsonString() throws IOException {
  // Given:
  final Schema msgSchema = SchemaBuilder.struct()
      .field("field1", Schema.STRING_SCHEMA)
      .field("field2", Schema.INT32_SCHEMA)
      .build();
  final Struct msgStruct = new Struct(msgSchema);
  msgStruct.put("field1", "foobar");
  msgStruct.put("field2", 123);
  final SchemaAndValue schemaAndValue = new SchemaAndValue(msgSchema, msgStruct);

  // When:
  logger.info(schemaAndValue);

  // Then:
  verify(innerLogger).info(any(), captor.capture());
  final String asString = captor.getValue().toString();
  final Object deserialized = new ObjectMapper().readValue(asString, Object.class);
  assertThat(deserialized, instanceOf(Map.class));
  assertThat(
      deserialized,
      equalTo(
          ImmutableMap.of(
              "field1", "foobar",
              "field2", 123)));
}