org.apache.kafka.connect.errors.DataException Java Examples

@Override
public WriteModel<BsonDocument> createWriteModel(final SinkDocument document) {
  BsonDocument vd =
      document
          .getValueDoc()
          .orElseThrow(
              () ->
                  new DataException(
                      "Error: cannot build the WriteModel since the value document was missing unexpectedly"));

  try {
    BsonDocument businessKey = vd.getDocument(ID_FIELD);
    vd.remove(ID_FIELD);
    return new ReplaceOneModel<>(businessKey, vd, REPLACE_OPTIONS);
  } catch (BSONException e) {
    throw new DataException(
        "Error: cannot build the WriteModel since the value document does not contain an _id field of"
            + " type BsonDocument which holds the business key fields");
  }
}
 

@Override
public WriteModel<BsonDocument> perform(final SinkDocument doc) {

  BsonDocument keyDoc =
      doc.getKeyDoc()
          .orElseThrow(
              () -> new DataException("Error: key doc must not be missing for insert operation"));

  BsonDocument valueDoc =
      doc.getValueDoc()
          .orElseThrow(
              () ->
                  new DataException("Error: value doc must not be missing for insert operation"));

  try {
    BsonDocument filterDoc =
        RdbmsHandler.generateFilterDoc(keyDoc, valueDoc, OperationType.CREATE);
    BsonDocument upsertDoc = RdbmsHandler.generateUpsertOrReplaceDoc(keyDoc, valueDoc, filterDoc);
    return new ReplaceOneModel<>(filterDoc, upsertDoc, REPLACE_OPTIONS);
  } catch (Exception exc) {
    throw new DataException(exc);
  }
}
 

@Override
public BsonValue toBson(final Object data) {

  // obviously SinkRecords may contain different types
  // to represent byte arrays
  if (data instanceof ByteBuffer) {
    return new BsonBinary(((ByteBuffer) data).array());
  }

  if (data instanceof byte[]) {
    return new BsonBinary((byte[]) data);
  }

  throw new DataException(
      "Error: bytes field conversion failed due to unexpected object type "
          + data.getClass().getName());
}
 

static BsonDocument generateUpsertOrReplaceDoc(
    final BsonDocument keyDoc, final BsonDocument valueDoc, final BsonDocument filterDoc) {

  if (!valueDoc.containsKey(JSON_DOC_AFTER_FIELD)
      || valueDoc.get(JSON_DOC_AFTER_FIELD).isNull()
      || !valueDoc.get(JSON_DOC_AFTER_FIELD).isDocument()
      || valueDoc.getDocument(JSON_DOC_AFTER_FIELD).isEmpty()) {
    throw new DataException(
        "Error: valueDoc must contain non-empty 'after' field"
            + " of type document for insert/update operation");
  }

  BsonDocument upsertDoc = new BsonDocument();
  if (filterDoc.containsKey(ID_FIELD)) {
    upsertDoc.put(ID_FIELD, filterDoc.get(ID_FIELD));
  }

  BsonDocument afterDoc = valueDoc.getDocument(JSON_DOC_AFTER_FIELD);
  for (String f : afterDoc.keySet()) {
    if (!keyDoc.containsKey(f)) {
      upsertDoc.put(f, afterDoc.get(f));
    }
  }
  return upsertDoc;
}
 

@Override
public WriteModel<BsonDocument> perform(final SinkDocument doc) {

  BsonDocument keyDoc =
      doc.getKeyDoc()
          .orElseThrow(
              () -> new DataException("Error: key doc must not be missing for delete operation"));

  BsonDocument valueDoc =
      doc.getValueDoc()
          .orElseThrow(
              () ->
                  new DataException("Error: value doc must not be missing for delete operation"));

  try {
    BsonDocument filterDoc =
        RdbmsHandler.generateFilterDoc(keyDoc, valueDoc, OperationType.DELETE);
    return new DeleteOneModel<>(filterDoc);
  } catch (Exception exc) {
    throw new DataException(exc);
  }
}
 

/**
 * Takes the {@link SinkRecord}'s value as a byte array.
 *
 * <p>If the value is {@code null}, it outputs nothing.
 *
 * <p>If the value is not {@code null}, it assumes the value <b>is</b> a byte array.
 *
 * @param record       the record to get the value from
 * @param outputStream the stream to write to
 * @throws DataException when the value is not actually a byte array
 */
@Override
public void write(final SinkRecord record,
                  final OutputStream outputStream) throws IOException {
    Objects.requireNonNull(record, "record cannot be null");
    Objects.requireNonNull(record.valueSchema(), "value schema cannot be null");
    Objects.requireNonNull(outputStream, "outputStream cannot be null");

    if (record.valueSchema().type() != Schema.Type.BYTES) {
        final String msg = String.format("Record value schema type must be %s, %s given",
            Schema.Type.BYTES, record.valueSchema().type());
        throw new DataException(msg);
    }

    // Do nothing if the key is null.
    if (record.value() == null) {
        return;
    }

    if (!(record.value() instanceof byte[])) {
        throw new DataException("Value is not a byte array");
    }

    outputStream.write(getOutputBytes((byte[]) record.value()));
}
 

public BsonValue toBson(final Object data, final Schema fieldSchema) {
  if (!fieldSchema.isOptional()) {
    if (data == null) {
      throw new DataException("Error: schema not optional but data was null");
    }
    LOGGER.trace("field not optional and data is '{}'", data.toString());
    return toBson(data);
  }

  if (data != null) {
    LOGGER.trace("field optional and data is '{}'", data.toString());
    return toBson(data);
  }

  if (fieldSchema.defaultValue() != null) {
    LOGGER.trace(
        "field optional and no data but default value is '{}'",
        fieldSchema.defaultValue().toString());
    return toBson(fieldSchema.defaultValue());
  }

  LOGGER.trace("field optional, no data and no default value thus '{}'", BsonNull.VALUE);
  return BsonNull.VALUE;
}
 

@Override
public BsonValue toBson(final Object data) {
  if (data instanceof BigDecimal) {
    if (format.equals(Format.DECIMAL128)) {
      return new BsonDecimal128(new Decimal128((BigDecimal) data));
    }
    if (format.equals(Format.LEGACYDOUBLE)) {
      return new BsonDouble(((BigDecimal) data).doubleValue());
    }
  }

  throw new DataException(
      "Error: decimal conversion not possible when data is of type "
          + data.getClass().getName()
          + " and format is "
          + format);
}
 

@Override
public WriteModel<BsonDocument> perform(final SinkDocument doc) {

  BsonDocument valueDoc =
      doc.getValueDoc()
          .orElseThrow(
              () ->
                  new DataException("Error: value doc must not be missing for insert operation"));

  try {
    BsonDocument insertDoc =
        BsonDocument.parse(valueDoc.get(JSON_DOC_FIELD_PATH).asString().getValue());
    return new ReplaceOneModel<>(
        new BsonDocument(ID_FIELD, insertDoc.get(ID_FIELD)), insertDoc, REPLACE_OPTIONS);
  } catch (Exception exc) {
    throw new DataException(exc);
  }
}
 

@Override
public Optional<WriteModel<BsonDocument>> handle(final SinkDocument doc) {

  BsonDocument keyDoc =
      doc.getKeyDoc()
          .orElseThrow(
              () -> new DataException("Error: key document must not be missing for CDC mode"));

  BsonDocument valueDoc = doc.getValueDoc().orElseGet(BsonDocument::new);

  if (keyDoc.containsKey(JSON_ID_FIELD) && valueDoc.isEmpty()) {
    LOGGER.debug("skipping debezium tombstone event for kafka topic compaction");
    return Optional.empty();
  }

  LOGGER.debug("key: " + keyDoc.toString());
  LOGGER.debug("value: " + valueDoc.toString());

  return Optional.of(getCdcOperation(valueDoc).perform(doc));
}
 

private RecordConverter getRecordConverter(final Object data, final Schema schema) {
  // AVRO or JSON with schema
  if (schema != null && data instanceof Struct) {
    LOGGER.debug("using schemaful converter");
    return schemafulConverter;
  }

  // structured JSON without schema
  if (data instanceof Map) {
    LOGGER.debug("using schemaless converter");
    return schemalessConverter;
  }

  // raw JSON string
  if (data instanceof String) {
    LOGGER.debug("using raw converter");
    return rawConverter;
  }

  throw new DataException(
      "Error: no converter present due to unexpected object type " + data.getClass().getName());
}
 

/**
 * MapEntry types in connect Schemas are represented as Arrays of record.
 * Return the array type from the union instead of the union itself.
 */
private static org.apache.avro.Schema avroSchemaForUnderlyingMapEntryType(
    Schema schema,
    org.apache.avro.Schema avroSchema) {

    if (schema != null && schema.isOptional()) {
        if (avroSchema.getType() == org.apache.avro.Schema.Type.UNION) {
            for (org.apache.avro.Schema typeSchema : avroSchema.getTypes()) {
                if (!typeSchema.getType().equals(org.apache.avro.Schema.Type.NULL)
                    && Schema.Type.ARRAY.getName().equals(typeSchema.getType().getName())) {
                    return typeSchema;
                }
            }
        } else {
            throw new DataException(
                "An optional schema should have an Avro Union type, not "
                + schema.type());
        }
    }
    return avroSchema;
}
 

@Override
public BsonValue generateId(final SinkDocument doc, final SinkRecord orig) {
  Optional<BsonDocument> optionalDoc = Optional.empty();
  if (where.equals(ProvidedIn.KEY)) {
    optionalDoc = doc.getKeyDoc();
  }

  if (where.equals(ProvidedIn.VALUE)) {
    optionalDoc = doc.getValueDoc();
  }

  BsonValue id =
      optionalDoc
          .map(d -> d.get(ID_FIELD))
          .orElseThrow(
              () ->
                  new DataException(
                      "Error: provided id strategy is used but the document structure either contained"
                          + " no _id field or it was null"));

  if (id instanceof BsonNull) {
    throw new DataException(
        "Error: provided id strategy used but the document structure contained an _id of type BsonNull");
  }
  return id;
}
 

private UUID constructUuidObjectFromString(final String uuid) {
  try {
    if (uuid.length() == UUID_LENGTH) {
      return UUID.fromString(uuid);
    } else if (uuid.length() == UUID_LENGTH_NO_DASHES) {
      return UUID.fromString(
          uuid.replaceFirst(
              "(\\p{XDigit}{8})(\\p{XDigit}{4})(\\p{XDigit}{4})(\\p{XDigit}{4})(\\p{XDigit}+)",
              "$1-$2-$3-$4-$5"));
    }
  } catch (Exception e) {
    // ignore
  }

  throw new DataException(format("UUID cannot be constructed from provided value: `%s`", uuid));
}
 

@Test
@DisplayName("when key doc and value 'before' field both empty then DataException")
void testEmptyKeyDocAndEmptyValueBeforeField() {
  assertThrows(
      DataException.class,
      () ->
          RDBMS_UPDATE.perform(
              new SinkDocument(
                  new BsonDocument(), new BsonDocument("before", new BsonDocument()))));
}
 

@Test
@DisplayName("when value doc contains unknown operation type then DataException")
void testUnkownCdcOperationType() {
  SinkDocument cdcEvent =
      new SinkDocument(BsonDocument.parse("{id: 1234}"), BsonDocument.parse("{op: 'x'}"));
  assertThrows(DataException.class, () -> RDBMS_HANDLER_DEFAULT_MAPPING.handle(cdcEvent));
}
 

@Test
@DisplayName("when value doc is null operation type then DataException")
void testNullCdcOperationType() {
  SinkDocument cdcEvent =
      new SinkDocument(BsonDocument.parse("{id: 1234}"), BsonDocument.parse("{op: null}"));
  assertThrows(DataException.class, () -> RDBMS_HANDLER_DEFAULT_MAPPING.handle(cdcEvent));
}
 

@Test
@DisplayName("when value doc contains unmapped operation type then DataException")
void testUnmappedCdcOperationType() {
  SinkDocument cdcEvent =
      new SinkDocument(
          BsonDocument.parse("{id: 1234}"),
          BsonDocument.parse("{op: 'c', after: {id: 1234, foo: 'bar'}}"));
  assertThrows(DataException.class, () -> RDBMS_HANDLER_EMPTY_MAPPING.handle(cdcEvent));
}
 

@Test
@DisplayName("when value doc is missing operation type then DataException")
void testMissingCdcOperationType() {
  SinkDocument cdcEvent =
      new SinkDocument(BsonDocument.parse("{id: 1234}"), BsonDocument.parse("{po: null}"));
  assertThrows(DataException.class, () -> RDBMS_HANDLER_DEFAULT_MAPPING.handle(cdcEvent));
}
 

@Test
@DisplayName("test avro or (json + schema) conversion (which is handled the same)")
void testAvroOrJsonWithSchemaConversion() {
  RecordConverter converter = new AvroJsonSchemafulRecordConverter();
  assertAll(
      "",
      () -> assertEquals(expectedBsonDocBytes1, converter.convert(objSchema1, objStruct1)),
      () -> assertThrows(DataException.class, () -> converter.convert(objSchema1, null)),
      () -> assertThrows(DataException.class, () -> converter.convert(null, objStruct1)),
      () -> assertThrows(DataException.class, () -> converter.convert(null, null)));
}
 

@TestFactory
@DisplayName("tests for string field conversions")
List<DynamicTest> testStringFieldConverter() {
  SinkFieldConverter converter = new StringFieldConverter();
  List<DynamicTest> tests = new ArrayList<>();

  asList("fooFOO", "", "blahBLAH")
      .forEach(
          el ->
              tests.add(
                  dynamicTest(
                      "conversion with " + converter.getClass().getSimpleName() + " for " + el,
                      () -> assertEquals(el, ((BsonString) converter.toBson(el)).getValue()))));

  tests.add(
      dynamicTest(
          "optional type conversions",
          () -> {
            Schema valueOptionalDefault = SchemaBuilder.string().optional().defaultValue("");
            assertAll(
                "checks",
                () ->
                    assertThrows(
                        DataException.class, () -> converter.toBson(null, Schema.STRING_SCHEMA)),
                () ->
                    assertEquals(
                        new BsonNull(), converter.toBson(null, Schema.OPTIONAL_STRING_SCHEMA)),
                () ->
                    assertEquals(
                        valueOptionalDefault.defaultValue(),
                        ((BsonString) converter.toBson(null, valueOptionalDefault)).getValue()));
          }));

  return tests;
}
 

@TestFactory
@DisplayName("tests for float64 field conversions")
List<DynamicTest> testFloat64FieldConverter() {
  SinkFieldConverter converter = new Float64FieldConverter();
  List<DynamicTest> tests = new ArrayList<>();

  asList(Double.MIN_VALUE, 0d, Double.MAX_VALUE)
      .forEach(
          el ->
              tests.add(
                  dynamicTest(
                      "conversion with " + converter.getClass().getSimpleName() + " for " + el,
                      () ->
                          assertEquals(
                              (double) el, ((BsonDouble) converter.toBson(el)).getValue()))));

  tests.add(
      dynamicTest(
          "optional type conversions",
          () -> {
            Schema valueOptionalDefault = SchemaBuilder.float64().optional().defaultValue(0.0d);
            assertAll(
                "checks",
                () ->
                    assertThrows(
                        DataException.class, () -> converter.toBson(null, Schema.FLOAT64_SCHEMA)),
                () ->
                    assertEquals(
                        new BsonNull(), converter.toBson(null, Schema.OPTIONAL_FLOAT64_SCHEMA)),
                () ->
                    assertEquals(
                        valueOptionalDefault.defaultValue(),
                        ((BsonDouble) converter.toBson(null, valueOptionalDefault)).getValue()));
          }));
  return tests;
}
 

@TestFactory
@DisplayName("tests for float32 field conversions")
List<DynamicTest> testFloat32FieldConverter() {
  SinkFieldConverter converter = new Float32FieldConverter();
  List<DynamicTest> tests = new ArrayList<>();

  asList(Float.MIN_VALUE, 0f, Float.MAX_VALUE)
      .forEach(
          el ->
              tests.add(
                  dynamicTest(
                      "conversion with " + converter.getClass().getSimpleName() + " for " + el,
                      () ->
                          assertEquals(
                              (float) el, ((BsonDouble) converter.toBson(el)).getValue()))));

  tests.add(
      dynamicTest(
          "optional type conversions",
          () -> {
            Schema valueOptionalDefault = SchemaBuilder.float32().optional().defaultValue(0.0f);
            assertAll(
                "checks",
                () ->
                    assertThrows(
                        DataException.class, () -> converter.toBson(null, Schema.FLOAT32_SCHEMA)),
                () ->
                    assertEquals(
                        new BsonNull(), converter.toBson(null, Schema.OPTIONAL_FLOAT32_SCHEMA)),
                () ->
                    assertEquals(
                        ((Float) valueOptionalDefault.defaultValue()).doubleValue(),
                        ((BsonDouble) converter.toBson(null, valueOptionalDefault)).getValue()));
          }));

  return tests;
}
 

@Test
@DisplayName(
    "when value document is missing for UpdateOneBusinessKeyTimestampStrategy then DataException")
void testUpdateOneBusinessKeyTimestampsStrategyWithMissingValueDocument() {
  assertThrows(
      DataException.class,
      () ->
          UPDATE_ONE_BUSINESS_KEY_TIMESTAMPS_STRATEGY.createWriteModel(
              new SinkDocument(new BsonDocument(), null)));
}
 

@TestFactory
@DisplayName("tests for int32 field conversions")
List<DynamicTest> testInt32FieldConverter() {

  SinkFieldConverter converter = new Int32FieldConverter();

  List<DynamicTest> tests = new ArrayList<>();
  asList(Integer.MIN_VALUE, 0, Integer.MAX_VALUE)
      .forEach(
          el ->
              tests.add(
                  dynamicTest(
                      "conversion with " + converter.getClass().getSimpleName() + " for " + el,
                      () ->
                          assertEquals(
                              (int) el, ((BsonInt32) converter.toBson(el)).getValue()))));

  tests.add(
      dynamicTest(
          "optional type conversions",
          () -> {
            Schema valueOptionalDefault = SchemaBuilder.int32().optional().defaultValue(0);
            assertAll(
                "checks",
                () ->
                    assertThrows(
                        DataException.class, () -> converter.toBson(null, Schema.INT32_SCHEMA)),
                () ->
                    assertEquals(
                        new BsonNull(), converter.toBson(null, Schema.OPTIONAL_INT32_SCHEMA)),
                () ->
                    assertEquals(
                        valueOptionalDefault.defaultValue(),
                        ((BsonInt32) converter.toBson(null, valueOptionalDefault)).getValue()));
          }));

  return tests;
}
 

@TestFactory
@DisplayName("tests for int16 field conversions")
List<DynamicTest> testInt16FieldConverter() {

  SinkFieldConverter converter = new Int16FieldConverter();

  List<DynamicTest> tests = new ArrayList<>();
  asList(Short.MIN_VALUE, (short) 0, Short.MAX_VALUE)
      .forEach(
          el ->
              tests.add(
                  dynamicTest(
                      "conversion with " + converter.getClass().getSimpleName() + " for " + el,
                      () ->
                          assertEquals(
                              (short) el, ((BsonInt32) converter.toBson(el)).getValue()))));

  tests.add(
      dynamicTest(
          "optional type conversions",
          () -> {
            Schema valueOptionalDefault =
                SchemaBuilder.int16().optional().defaultValue((short) 0);
            assertAll(
                "checks",
                () ->
                    assertThrows(
                        DataException.class, () -> converter.toBson(null, Schema.INT16_SCHEMA)),
                () ->
                    assertEquals(
                        new BsonNull(), converter.toBson(null, Schema.OPTIONAL_INT16_SCHEMA)),
                () ->
                    assertEquals(
                        ((short) valueOptionalDefault.defaultValue()),
                        ((BsonInt32) converter.toBson(null, valueOptionalDefault)).getValue()));
          }));

  return tests;
}
 

@TestFactory
@DisplayName("tests for int8 field conversions")
List<DynamicTest> testInt8FieldConverter() {

  SinkFieldConverter converter = new Int8FieldConverter();

  List<DynamicTest> tests = new ArrayList<>();
  asList(Byte.MIN_VALUE, (byte) 0, Byte.MAX_VALUE)
      .forEach(
          el ->
              tests.add(
                  dynamicTest(
                      "conversion with " + converter.getClass().getSimpleName() + " for " + el,
                      () ->
                          assertEquals(
                              (int) el, ((BsonInt32) converter.toBson(el)).getValue()))));

  tests.add(
      dynamicTest(
          "optional type conversions",
          () -> {
            Schema valueOptionalDefault = SchemaBuilder.int8().optional().defaultValue((byte) 0);
            assertAll(
                "checks",
                () ->
                    assertThrows(
                        DataException.class, () -> converter.toBson(null, Schema.INT8_SCHEMA)),
                () ->
                    assertEquals(
                        new BsonNull(), converter.toBson(null, Schema.OPTIONAL_INT8_SCHEMA)),
                () ->
                    assertEquals(
                        ((Byte) valueOptionalDefault.defaultValue()).intValue(),
                        ((BsonInt32) converter.toBson(null, valueOptionalDefault)).getValue()));
          }));

  return tests;
}
 

@TestFactory
@DisplayName("tests for boolean field conversions")
List<DynamicTest> testBooleanFieldConverter() {

  SinkFieldConverter converter = new BooleanFieldConverter();

  List<DynamicTest> tests = new ArrayList<>();
  asList(true, false)
      .forEach(
          el ->
              tests.add(
                  dynamicTest(
                      "conversion with " + converter.getClass().getSimpleName() + " for " + el,
                      () -> assertEquals(el, ((BsonBoolean) converter.toBson(el)).getValue()))));

  tests.add(
      dynamicTest(
          "optional type conversion checks",
          () -> {
            Schema valueOptionalDefault = SchemaBuilder.bool().optional().defaultValue(true);
            assertAll(
                "",
                () ->
                    assertThrows(
                        DataException.class, () -> converter.toBson(null, Schema.BOOLEAN_SCHEMA)),
                () ->
                    assertEquals(
                        new BsonNull(), converter.toBson(null, Schema.OPTIONAL_BOOLEAN_SCHEMA)),
                () ->
                    assertEquals(
                        valueOptionalDefault.defaultValue(),
                        converter.toBson(null, valueOptionalDefault).asBoolean().getValue()));
          }));

  return tests;
}
 

@Test
@DisplayName("test json object conversion")
void testJsonObjectConversion() {
  RecordConverter converter = new JsonSchemalessRecordConverter();
  assertAll(
      "",
      () -> assertEquals(expectedBsonDocBytes1, converter.convert(null, objMap1)),
      () -> assertThrows(DataException.class, () -> converter.convert(null, null)));
}
 

@Override
public BsonValue generateId(final SinkDocument doc, final SinkRecord orig) {
  BsonValue id = super.generateId(doc, orig);

  if (id.isBinary() && BsonBinarySubType.isUuid(id.asBinary().getType())) {
    return id;
  } else if (id.isString()) {
    return new BsonBinary(
        constructUuidObjectFromString(id.asString().getValue()), UuidRepresentation.STANDARD);
  }

  throw new DataException(format("UUID cannot be constructed from provided value: `%s`", id));
}