org.apache.kafka.connect.transforms.Transformation Java Examples

@Test
public void testIfItConvertsConnectRecordCorrectly() {
    final SourceRecord connectRecord = new SourceRecord(Collections.emptyMap(), Collections.emptyMap(), "topic", Schema.STRING_SCHEMA, "1234", Schema.STRING_SCHEMA, "TRUE");

    final Map<String, Object> propsForKeySmt = new HashMap<>();
    propsForKeySmt.put(CamelTypeConverterTransform.FIELD_TARGET_TYPE_CONFIG, Integer.class.getName());

    final Map<String, Object> propsForValueSmt = new HashMap<>();
    propsForValueSmt.put(CamelTypeConverterTransform.FIELD_TARGET_TYPE_CONFIG, "java.lang.Boolean");

    final Transformation<SourceRecord> transformationKey = new CamelTypeConverterTransform.Key<>();
    final Transformation<SourceRecord> transformationValue = new CamelTypeConverterTransform.Value<>();

    transformationKey.configure(propsForKeySmt);
    transformationValue.configure(propsForValueSmt);

    final SourceRecord transformedKeySourceRecord = transformationKey.apply(connectRecord);
    final SourceRecord transformedValueSourceRecord = transformationValue.apply(connectRecord);

    assertEquals(1234, transformedKeySourceRecord.key());
    assertEquals(Schema.INT32_SCHEMA, transformedKeySourceRecord.keySchema());

    assertEquals(true, transformedValueSourceRecord.value());
    assertEquals(Schema.BOOLEAN_SCHEMA, transformedValueSourceRecord.valueSchema());
}
 

Set<Class<? extends Transformation>> findTransformations() {
  Set<Class<? extends Transformation>> transforms = this.reflections.getSubTypesOf(Transformation.class)
      .stream()
      .filter(c -> c.getName().startsWith(pkg.getName()))
      .filter(c -> Modifier.isPublic(c.getModifiers()))
      .filter(c -> !Modifier.isAbstract(c.getModifiers()))
      .filter((Predicate<Class<? extends Transformation>>) aClass -> Arrays.stream(aClass.getConstructors())
          .filter(c -> Modifier.isPublic(c.getModifiers()))
          .anyMatch(c -> c.getParameterCount() == 0))
      .sorted(Comparator.comparing(Class::getName))
      .collect(Collectors.toCollection(LinkedHashSet::new));
  final Set<Class<? extends Transformation>> result = new LinkedHashSet<>();


  for (Class<? extends Transformation> cls : transforms) {
    log.trace("findTransformations() - simpleName = '{}'", cls.getSimpleName());
    if (KEY_OR_VALUE.contains(cls.getSimpleName()) && null != cls.getDeclaringClass()) {
      result.add(cls.getDeclaringClass().asSubclass(Transformation.class));
    } else {
      result.add(cls);
    }
  }

  return result;
}
 

private List<Transformation<SourceRecord>> validateTransformations(
    List<Transformation<SourceRecord>> transformations) {
  List<Transformation<SourceRecord>> regexRouters = new ArrayList<>();

  // No need to validate transforms if we're not checking destination partitions
  if (this.topicCheckingEnabled) {
    for (Transformation<SourceRecord> transform : transformations) {
      String transformName = transform.getClass().getSimpleName();
      if (transform instanceof RegexRouter) {
        regexRouters.add(transform);
        // Slightly awkward check to see if any other routing transforms are configured
      } else if (transformName.contains("Router")) {
        throw new IllegalArgumentException(
            String.format(
                "Unsupported Router Transformation %s found."
                    + " To use it, please disable destination topic checking by setting 'enable.destination.topic.checking' to false.",
                transformName));
      } else {
        logger.debug("Ignoring non-routing Transformation {}", transformName);
      }
    }
  }
  return regexRouters;
}
 

private String applyRoutersToTopic(String topic) {
  TopicPartition topicPartition = new TopicPartition(topic, 0);
  Map<String, Object> sourcePartition = TopicPartitionSerDe.asMap(topicPartition);
  SourceRecord record =
      new SourceRecord(
          sourcePartition,
          null,
          topicPartition.topic(),
          topicPartition.partition(),
          Schema.BYTES_SCHEMA,
          null,
          Schema.OPTIONAL_BYTES_SCHEMA,
          null);
  for (Transformation<SourceRecord> transform : this.routers) {
    record = transform.apply(record);
  }
  return record.topic();
}
 

@Test
public void testIfHandlesTypeConvertersFromCamelComponents() {
    // we know we have a type converter from struct to map in dbz component, so we use this for testing
    final Schema schema = SchemaBuilder.struct()
            .field("id", Schema.INT32_SCHEMA)
            .field("name", Schema.STRING_SCHEMA)
            .field("valid", Schema.BOOLEAN_SCHEMA)
            .field("extra", Schema.STRING_SCHEMA)
            .build();

    final Struct value = new Struct(schema);
    value.put("id", 12);
    value.put("name", "test-name");
    value.put("valid", true);

    final SourceRecord connectRecord = new SourceRecord(Collections.emptyMap(), Collections.emptyMap(), "topic", Schema.STRING_SCHEMA, "1234", schema, value);

    final Map<String, Object> props = new HashMap<>();
    props.put(CamelTypeConverterTransform.FIELD_TARGET_TYPE_CONFIG, Map.class.getName());

    final Transformation<SourceRecord> transformationValue = new CamelTypeConverterTransform.Value<>();

    transformationValue.configure(props);

    final SourceRecord transformedValueSourceRecord = transformationValue.apply(connectRecord);

    // assert
    assertNotNull(transformedValueSourceRecord);

    final Map<String, Object> outputValue = (Map<String, Object>) transformedValueSourceRecord.value();

    assertEquals(12, outputValue.get("id"));
    assertEquals("test-name", outputValue.get("name"));
    assertNull(outputValue.get("extra"));
    assertTrue((boolean)outputValue.get("valid"));
    assertEquals(Schema.Type.MAP, transformedValueSourceRecord.valueSchema().type());
}
 

@Test
public void findTransformations() {
  final Set<Class<? extends Transformation>> expected = ImmutableSet.of(
      NoDocTestTransformation.class,
      TestTransformation.class,
      ToUpperCase.class,
      TestKeyAndValueTransformation.class
  );
  final Set<Class<? extends Transformation>> actual = pluginLoader.findTransformations();

  assertEquals(expected, actual);
}
 

public Plugin load() {
  ImmutablePlugin.Builder builder = ImmutablePlugin.builder()
      .from(notes(this.pkg))
      .pluginName(AnnotationHelper.pluginName(this.pkg))
      .pluginOwner(AnnotationHelper.pluginOwner(this.pkg));
  List<Plugin.Transformation> transformations = loadTransformations();
  builder.addAllTransformations(transformations);
  List<Plugin.SinkConnector> sinkConnectors = loadSinkConnectors();
  builder.addAllSinkConnectors(sinkConnectors);
  List<Plugin.SourceConnector> sourceConnectors = loadSourceConnectors();
  builder.addAllSourceConnectors(sourceConnectors);
  List<Plugin.Converter> converters = loadConverters();
  builder.addAllConverters(converters);
  return builder.build();
}
 

ConfigDef transformationConfig(Class<? extends Transformation> transformation) {
  try {
    return transformation.newInstance().config();
  } catch (InstantiationException | IllegalAccessException e) {
    throw new IllegalStateException(e);
  }
}
 

@Test
public void testIfItCanHandleEmptyKeyProps() {
    final Transformation<SourceRecord> transformationKey = new CamelTypeConverterTransform.Key<>();

    final Map<String, Object> props = new HashMap<>();
    props.put(CamelTypeConverterTransform.FIELD_TARGET_TYPE_CONFIG, Map.class.getName());

    assertThrows(ConfigException.class, () -> transformationKey.configure(Collections.emptyMap()));
}
 

private List<Transformation<SourceRecord>> buildTransformations() {
  List<Transformation<SourceRecord>> transformations = new ArrayList<>();
  List<String> transformNames = simpleConfig.getList(ConnectorConfig.TRANSFORMS_CONFIG);
  for (String name : transformNames) {
    String configPrefix = ConnectorConfig.TRANSFORMS_CONFIG + "." + name + ".";

    // We don't have access to Plugins to properly add loaded classes' configs to the definition,
    // so retrieve it based on the transform prefix.
    Map<String, Object> transformConfig = simpleConfig.originalsWithPrefix(configPrefix);
    String transformClassName = (String) transformConfig.get("type");

    Transformation<SourceRecord> transform;
    try {
      Class<?> transformClass =
          (Class<?>)
              ConfigDef.parseType(
                  configPrefix + "type", transformClassName, ConfigDef.Type.CLASS);
      transform = transformClass.asSubclass(Transformation.class).newInstance();
      transform.configure(transformConfig);
    } catch (RuntimeException | InstantiationException | IllegalAccessException e) {
      // If we couldn't build and configure the Transformation properly we can't verify
      // that we'll be looking for the right target topics, so throw an error.
      throw new ConnectException(
          String.format("Error building transformation %s from config", name), e);
    }
    transformations.add(transform);
  }
  return transformations;
}
 

@Test
public void transformationsShouldBeAvailable() {
  Map<String, String> properties = new HashMap<>();
  properties.put("name", "connector");
  properties.put("transforms", "a");
  properties.put("transforms.a.type", SimpleTransformation.class.getName());
  properties.put("transforms.a.magic.number", "45");
  SourceConfig configWithTransform = new SourceConfig(properties);

  List<Transformation<SourceRecord>> transformations = configWithTransform.transformations();

  SimpleTransformation<SourceRecord> expectedTransform = new SimpleTransformation<>();
  expectedTransform.configure(Collections.singletonMap("magic.number", "45"));
  assertThat(transformations, contains(expectedTransform));
}
 

@Override
protected Transformation<SinkRecord> create() {
  return new ToLong.Value<>();
}
 

private List<Plugin.Transformation> loadTransformations() {
  if (null != this.transformations) {
    return this.transformations;
  }
  List<Plugin.Transformation> result = new ArrayList<>();
  Set<Class<? extends Transformation>> tranformations = findTransformations();

  for (Class<? extends Transformation> cls : tranformations) {
    log.trace("loadTransformations() - processing {}", cls.getName());
    ImmutableTransformation.Builder builder = ImmutableTransformation.builder()
        .cls(cls)
        .from(notes(cls));
    Class[] classes = cls.getClasses();
    boolean isKeyValue = false;
    Class keyClass = null;
    Class valueClass = null;

    if (null != classes) {
      for (Class c : classes) {
        if ("Key".equals(c.getSimpleName())) {
          keyClass = c;
          isKeyValue = true;
        } else if ("Value".equals(c.getSimpleName())) {
          isKeyValue = true;
          valueClass = c;
        }
      }
    } else {
      isKeyValue = false;
    }
    builder.isKeyValue(isKeyValue);
    if (null != keyClass) {
      builder.key(keyClass);
    }
    if (null != valueClass) {
      builder.value(valueClass);
    }
    ConfigDef configDef;
    if (isKeyValue) {
      if (null != keyClass) {
        configDef = transformationConfig(keyClass);
      } else if (null != valueClass) {
        configDef = transformationConfig(valueClass);
      } else {
        throw new IllegalStateException("key and value class null");
      }
    } else {
      configDef = transformationConfig(cls);
    }
    Plugin.Configuration configuration = config(configDef);
    builder.configuration(configuration);

    List<String> examples = findExamples(cls);
    builder.addAllExamples(examples);

    result.add(builder.build());
  }

  return (this.transformations = result);
}
 

public List<Transformation<SourceRecord>> transformations() {
  if (this.transformations == null) {
    this.transformations = buildTransformations();
  }
  return this.transformations;
}
 

@TestFactory
public Stream<DynamicTest> validateTransformationExamples() {
  Map<File, Plugin.Transformation> transformationExamples = examples(this.plugin.getTransformations());

  return transformationExamples.entrySet().stream()
      .map(e -> dynamicTest(String.format("%s/%s", e.getValue().getCls().getSimpleName(), e.getKey().getName()), () -> {
        final Plugin.Transformation transformation = e.getValue();
        final File rstOutputFile = outputRST(
            this.transformationsExampleDirectory,
            transformation.getCls(),
            e.getKey()
        );
        final Plugin.TransformationExample example = loadExample(e, Plugin.TransformationExample.class);

        final Class<? extends Transformation> transformationClass;

        if (Strings.isNullOrEmpty(example.getChildClass())) {
          transformationClass = transformation.getCls();
        } else {
          Optional<Class> childClass = Arrays.stream(transformation.getCls().getClasses())
              .filter(c -> example.getChildClass().equals(c.getSimpleName()))
              .findFirst();
          assertTrue(
              childClass.isPresent(),
              String.format(
                  "Could not find child '%s' of class '%s'",
                  example.getChildClass(),
                  transformation.getCls().getName()
              )
          );
          transformationClass = childClass.get();
        }

        Transformation<SinkRecord> transform = transformationClass.newInstance();
        transform.configure(example.getConfig());

        ImmutableTransformationExampleInput.Builder builder = ImmutableTransformationExampleInput.builder();
        builder.example(example);

        if (null != example.getConfig() && !example.getConfig().isEmpty()) {
          String transformKey = CaseFormat.UPPER_CAMEL.to(
              CaseFormat.LOWER_CAMEL,
              transformation.getCls().getSimpleName()
          );
          String transformPrefix = "transforms." + transformKey + ".";
          LinkedHashMap<String, String> config = new LinkedHashMap<>();
          config.put("transforms", transformKey);
          config.put(transformPrefix + "type", transformationClass.getName());

          for (Map.Entry<String, String> a : example.getConfig().entrySet()) {
            config.put(transformPrefix + a.getKey(), a.getValue());
          }

          String configJson = writeValueAsIndentedString(config);
          builder.config(configJson);
        }

        if (null != example.getInput()) {
          String inputJson = writeValueAsIndentedString(example.getInput());
          builder.inputJson(inputJson);

          SinkRecord output = transform.apply(example.getInput());
          if (null != output) {
            String outputJson = writeValueAsIndentedString(output);
            builder.outputJson(outputJson);

            final List<String> inputLines = Arrays.asList(inputJson.split("\\r?\\n"));
            final List<String> outputLines = Arrays.asList(outputJson.split("\\r?\\n"));


            Patch<String> patch = DiffUtils.diff(inputLines, outputLines);
            for (AbstractDelta<String> delta : patch.getDeltas()) {
              log.trace("delta: start={} end={}", delta.getTarget().getPosition(), delta.getTarget().last());
              final int lineStart = delta.getTarget().getPosition() + 1;
              final int lineEnd = lineStart + delta.getTarget().size() - 1;
              for (int i = lineStart; i <= lineEnd; i++) {
                builder.addOutputEmphasizeLines(i);
              }
            }
          }
        }

        try (Writer writer = Files.newWriter(rstOutputFile, Charsets.UTF_8)) {
          Template template = configuration.getTemplate("rst/transformationExample.rst.ftl");
          process(writer, template, builder.build());
        }
      }));
}
 

@Override
protected Transformation<SinkRecord> create() {
  return new ExtractNestedField.Value();
}
 

@Override
protected Transformation<SinkRecord> create() {
  return new ExtractTimestamp.Value<>();
}
 

@Override
protected Transformation<SinkRecord> create() {
  return new ChangeCase.Value<>();
}
 

@Override
protected Transformation<SinkRecord> create() {
  return new BytesToString.Value<>();
}
 

@Override
protected Transformation<SinkRecord> create() {
  return new PatternRename.Value();
}
 

@Override
protected Transformation<SinkRecord> create() {
  return new PatternRename.Key();
}
 

@Override
protected Transformation<SinkRecord> create() {
  return new ChangeTopicCase<>();
}
 

@Override
protected Transformation<SinkRecord> create() {
  return new ToJSON.Value<>();
}
 

@Override
protected Transformation<SinkRecord> create() {
  return new TopicNameToField.Value<>();
}
 

protected abstract Transformation<SinkRecord> create();