Java Code Examples for org.apache.kafka.connect.data.Struct#getStruct()

@Override
protected void checkData(Struct record, long index) {
    Struct subrecord = record.getStruct(FIELD_STRUCT);
    assertAll(
            () -> assertEquals((int) (Integer) record.get(FIELD_INTEGER), index),
            () -> assertEquals((long) (Long) record.get(FIELD_LONG), Long.MAX_VALUE),
            () -> assertTrue(record.get(FIELD_STRING).toString().startsWith(index + "_")),
            () -> assertTrue(Boolean.parseBoolean(record.get(FIELD_BOOLEAN).toString())),
            () -> assertEquals((Double) record.get(FIELD_DECIMAL), Double.parseDouble(index + "." + index), 0),
            () -> assertNull(record.get(FIELD_NULL)),
            () -> assertNotNull(record.schema().field(FIELD_NULL)),
            () -> assertEquals(record.get(FIELD_ARRAY), Arrays.asList("elm[" + index + "]", "elm[" + (index + 1) + "]")),
            () -> assertEquals((int) (Integer) subrecord.get(FIELD_INTEGER), index),
            () -> assertEquals((long) (Long) subrecord.get(FIELD_LONG), Long.MAX_VALUE),
            () -> assertTrue(subrecord.get(FIELD_STRING).toString().startsWith(index + "_")),
            () -> assertTrue(Boolean.parseBoolean(subrecord.get(FIELD_BOOLEAN).toString())),
            () -> assertEquals((Double) subrecord.get(FIELD_DECIMAL), Double.parseDouble(index + "." + index), 0),
            () -> assertNull(subrecord.get(FIELD_NULL)),
            () -> assertNotNull(subrecord.schema().field(FIELD_NULL))
    );
}
 

void copy(Struct input, Struct output) {
  for (Field outputField : output.schema().fields()) {
    Field inputField = input.schema().field(outputField.name());
    if (null != inputField) {
      if (Schema.Type.STRUCT == outputField.schema().type()) {
        Struct inputStruct = input.getStruct(inputField.name());
        if (null == inputStruct) {
          output.put(outputField, null);
        } else {
          Struct outputStruct = new Struct(outputField.schema());
          copy(inputStruct, outputStruct);
        }
      } else {
        output.put(outputField, input.get(outputField.name()));
      }
    } else {
      log.trace("copy() - Skipping '{}' because input does not have field.", outputField.name());
    }
  }
}
 

public static Map<String, Object> asMap(Struct struct) {
  Preconditions.checkNotNull(struct, "struct cannot be null.");
  Map<String, Object> result = new LinkedHashMap<>(struct.schema().fields().size());

  for (Field field : struct.schema().fields()) {
    final Object value;
    if (Schema.Type.STRUCT == field.schema().type()) {
      Struct s = struct.getStruct(field.name());
      value = asMap(s);
    } else {
      value = struct.get(field);
    }
    result.put(field.name(), value);
  }

  return result;
}
 

@Override
public Instant getEventTimestamp(DataCollectionId source, OffsetContext offset, Object key, Struct value) {
    if (value == null) {
        return null;
    }
    final Struct sourceInfo = value.getStruct(Envelope.FieldName.SOURCE);
    if (source == null) {
        return null;
    }
    final Long timestamp = sourceInfo.getInt64(SourceInfo.TIMESTAMP_KEY);
    return timestamp == null ? null : Instant.ofEpochMilli(timestamp);
}
 

@Override
public Map<String, String> getEventSourcePosition(DataCollectionId source, OffsetContext offset, Object key, Struct value) {
    if (value == null) {
        return null;
    }
    final Struct sourceInfo = value.getStruct(Envelope.FieldName.SOURCE);
    if (source == null) {
        return null;
    }
    return Collect.hashMapOf(
            SourceInfo.COMMIT_LSN_KEY, sourceInfo.getString(SourceInfo.COMMIT_LSN_KEY),
            SourceInfo.CHANGE_LSN_KEY, sourceInfo.getString(SourceInfo.CHANGE_LSN_KEY));
}
 

@Override
public String getTransactionId(DataCollectionId source, OffsetContext offset, Object key, Struct value) {
    if (value == null) {
        return null;
    }
    final Struct sourceInfo = value.getStruct(Envelope.FieldName.SOURCE);
    if (source == null) {
        return null;
    }
    return sourceInfo.getString(SourceInfo.COMMIT_LSN_KEY);
}
 

@Override
public Instant getEventTimestamp(DataCollectionId source, OffsetContext offset, Object key, Struct value) {
    if (value == null) {
        return null;
    }
    final Struct sourceInfo = value.getStruct(Envelope.FieldName.SOURCE);
    if (source == null) {
        return null;
    }
    final Long timestamp = sourceInfo.getInt64(SourceInfo.TIMESTAMP_KEY);
    return timestamp == null ? null : Instant.ofEpochMilli(timestamp);
}
 

@Override
public Map<String, String> getEventSourcePosition(DataCollectionId source, OffsetContext offset, Object key, Struct value) {
    if (value == null) {
        return null;
    }
    final Struct sourceInfo = value.getStruct(Envelope.FieldName.SOURCE);
    if (source == null) {
        return null;
    }
    final Long scn = sourceInfo.getInt64(SourceInfo.SCN_KEY);
    return Collect.hashMapOf(
            SourceInfo.SCN_KEY, scn == null ? "null" : Long.toString(scn));
}
 

@Override
public String getTransactionId(DataCollectionId source, OffsetContext offset, Object key, Struct value) {
    if (value == null) {
        return null;
    }
    final Struct sourceInfo = value.getStruct(Envelope.FieldName.SOURCE);
    if (source == null) {
        return null;
    }
    return sourceInfo.getString(SourceInfo.TXID_KEY);
}
 

public Row translate(SourceRecord record) {
  LOG.debug("Source Record from Debezium: {}", record);

  String qualifiedTableName = record.topic();

  Struct recordValue = (Struct) record.value();
  if (recordValue == null) {
    return null;
  }

  // TODO: Consider including before value in the Row.
  Struct afterValue = recordValue.getStruct("after");
  Row afterValueRow = afterValue == null ? null : handleValue(afterValue.schema(), afterValue);
  LOG.debug("Beam Row is {}", afterValueRow);

  Row primaryKey = null;
  boolean hasPK = true;
  if (record.key() == null) {
    hasPK = false;
  } else {
    primaryKey = handleValue(record.keySchema(), record.key());
    LOG.debug("Key Schema: {} | Key Value: {}", primaryKey.getSchema(), primaryKey);
  }

  String sourceRecordOp = recordValue.getString("op");
  String operation = translateOperation(sourceRecordOp);
  if (operation == null) {
    return null;
  }

  Long timestampMs = recordValue.getInt64("ts_ms");

  if (!knownSchemas.containsKey(qualifiedTableName)) {
    org.apache.beam.sdk.schemas.Schema.Builder schemaBuilder = org.apache.beam.sdk.schemas.Schema
        .builder()
        .addStringField(DataflowCdcRowFormat.OPERATION)
        .addStringField(DataflowCdcRowFormat.TABLE_NAME)
        .addField(org.apache.beam.sdk.schemas.Schema.Field.nullable(
            DataflowCdcRowFormat.FULL_RECORD, FieldType.row(afterValueRow.getSchema())))
        .addInt64Field(DataflowCdcRowFormat.TIMESTAMP_MS);

    if (hasPK) {
      schemaBuilder.addRowField(DataflowCdcRowFormat.PRIMARY_KEY, primaryKey.getSchema());
    }
    knownSchemas.put(qualifiedTableName, schemaBuilder.build());
  }
  org.apache.beam.sdk.schemas.Schema finalBeamSchema = knownSchemas.get(qualifiedTableName);

  Row.Builder beamRowBuilder = Row.withSchema(finalBeamSchema)
      .addValue(operation)
      .addValue(qualifiedTableName)
      .addValue(afterValueRow)
      .addValue(timestampMs);

  if (hasPK) {
    beamRowBuilder.addValue(primaryKey);
  }

  return beamRowBuilder.build();
}