org.apache.kafka.connect.storage.Converter Java Examples

private static MirusOffsetTool newOffsetTool(Args args) throws IOException {
  // This needs to be the admin topic properties.
  // By default these are in the worker properties file, as this has the has admin producer and
  // consumer settings.  Separating these might be wise - also useful for storing state in
  // source cluster if it proves necessary.
  final Map<String, String> properties =
      !args.propertiesFile.isEmpty()
          ? Utils.propsToStringMap(Utils.loadProps(args.propertiesFile))
          : Collections.emptyMap();
  final DistributedConfig config = new DistributedConfig(properties);
  final KafkaOffsetBackingStore offsetBackingStore = new KafkaOffsetBackingStore();
  offsetBackingStore.configure(config);

  // Avoid initializing the entire Kafka Connect plugin system by assuming the
  // internal.[key|value].converter is org.apache.kafka.connect.json.JsonConverter
  final Converter internalConverter = new JsonConverter();
  internalConverter.configure(config.originalsWithPrefix("internal.key.converter."), true);

  final OffsetSetter offsetSetter = new OffsetSetter(internalConverter, offsetBackingStore);
  final OffsetFetcher offsetFetcher = new OffsetFetcher(config, internalConverter);
  final OffsetSerDe offsetSerDe = OffsetSerDeFactory.create(args.format);

  return new MirusOffsetTool(args, offsetFetcher, offsetSetter, offsetSerDe);
}
 

/**
 * Configure this class.
 * 
 * @param props initial configuration
 *
 * @throws ConnectException   Operation failed and connector should stop.
 */
public void configure(Map<String, String> props) {
    log.trace("[{}] Entry {}.configure, props={}", Thread.currentThread().getId(), this.getClass().getName(), props);

    super.configure(props);

    String converterClass = props.get(MQSinkConnector.CONFIG_NAME_MQ_MESSAGE_BUILDER_VALUE_CONVERTER);

    try {
        Class<? extends Converter> c = Class.forName(converterClass).asSubclass(Converter.class);
        converter = c.newInstance();

        // Make a copy of the configuration to filter out only those that begin "mq.message.builder.value.converter."
        // since those are used to configure the converter itself
        AbstractConfig ac = new AbstractConfig(new ConfigDef(), props, false);

        // Configure the Converter to convert the value, not the key (isKey == false)
        converter.configure(ac.originalsWithPrefix(MQSinkConnector.CONFIG_NAME_MQ_MESSAGE_BUILDER_VALUE_CONVERTER + "."), false);
    }
    catch (ClassNotFoundException | IllegalAccessException | InstantiationException | NullPointerException exc) {
        log.error("Could not instantiate converter for message builder {}", converterClass);
        throw new ConnectException("Could not instantiate converter for message builder", exc);
    }

    log.trace("[{}]  Exit {}.configure", Thread.currentThread().getId(), this.getClass().getName());
}
 

public Converter getKeyConverter() {
  Map<String, Object> conf = simpleConfig.originals();
  conf.put(StringConverterConfig.TYPE_CONFIG, ConverterType.KEY.getName());

  SimpleConfig config = new SimpleConfig(TaskConfigDefinition.configDef(), conf);

  return config.getConfiguredInstance(
      SourceConfigDefinition.SOURCE_KEY_CONVERTER.key, Converter.class);
}
 

public Converter getValueConverter() {
  Map<String, Object> conf = simpleConfig.originals();
  conf.put(StringConverterConfig.TYPE_CONFIG, ConverterType.VALUE.getName());

  SimpleConfig config = new SimpleConfig(TaskConfigDefinition.configDef(), conf);

  return config.getConfiguredInstance(
      SourceConfigDefinition.SOURCE_VALUE_CONVERTER.key, Converter.class);
}
 

OffsetFetcher(final WorkerConfig config, Converter internalConverter) {
  String topic = config.getString(DistributedConfig.OFFSET_STORAGE_TOPIC_CONFIG);
  if ("".equals(topic)) {
    throw new ConfigException("Offset storage topic must be specified");
  }

  Map<String, Object> producerProps = new HashMap<>(config.originals());
  producerProps.put(
      ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class.getName());
  producerProps.put(
      ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class.getName());
  producerProps.put(ProducerConfig.RETRIES_CONFIG, Integer.MAX_VALUE);

  Map<String, Object> consumerProps = new HashMap<>(config.originals());
  consumerProps.put(
      ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class.getName());
  consumerProps.put(
      ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class.getName());

  Callback<ConsumerRecord<byte[], byte[]>> consumedCallback =
      (error, record) -> {
        ByteBuffer key = record.key() != null ? ByteBuffer.wrap(record.key()) : null;
        ByteBuffer value = record.value() != null ? ByteBuffer.wrap(record.value()) : null;
        data.put(key, value);
      };
  this.offsetLog =
      new KafkaBasedLog<>(
          topic, producerProps, consumerProps, consumedCallback, Time.SYSTEM, null);
  this.internalConverter = internalConverter;
}
 

public Converter getKeyConverter() throws CassandraConnectorConfigException {
    try {
        Class keyConverterClass = Class.forName(this.getConfig().getString(KEY_CONVERTER_CLASS_CONFIG));
        Converter keyConverter = (Converter) keyConverterClass.newInstance();
        Map<String, Object> keyConverterConfigs = keyValueConverterConfigs(KEY_CONVERTER_PREFIX);
        keyConverter.configure(keyConverterConfigs, true);
        return keyConverter;
    }
    catch (Exception e) {
        throw new CassandraConnectorConfigException(e);
    }
}
 

public Converter getValueConverter() throws CassandraConnectorConfigException {
    try {
        Class valueConverterClass = Class.forName(this.getConfig().getString(VALUE_CONVERTER_CLASS_CONFIG));
        Converter valueConverter = (Converter) valueConverterClass.newInstance();
        Map<String, Object> valueConverterConfigs = keyValueConverterConfigs(VALUE_CONVERTER_PREFIX);
        valueConverter.configure(valueConverterConfigs, false);
        return valueConverter;
    }
    catch (Exception e) {
        throw new CassandraConnectorConfigException(e);
    }
}
 

public KafkaRecordEmitter(String kafkaTopicPrefix, String heartbeatPrefix, Properties kafkaProperties,
                          OffsetWriter offsetWriter, Duration offsetFlushIntervalMs, long maxOffsetFlushSize,
                          Converter keyConverter, Converter valueConverter) {
    this.producer = new KafkaProducer<>(kafkaProperties);
    this.topicSelector = CassandraTopicSelector.defaultSelector(kafkaTopicPrefix, heartbeatPrefix);
    this.offsetWriter = offsetWriter;
    this.offsetFlushPolicy = offsetFlushIntervalMs.isZero() ? OffsetFlushPolicy.always() : OffsetFlushPolicy.periodic(offsetFlushIntervalMs, maxOffsetFlushSize);
    this.keyConverter = keyConverter;
    this.valueConverter = valueConverter;
}
 

Set<Class<? extends Converter>> findConverters() {
  return this.reflections.getSubTypesOf(Converter.class)
      .stream()
      .filter(c -> c.getName().startsWith(pkg.getName()))
      .filter(c -> Modifier.isPublic(c.getModifiers()))
      .filter(c -> !Modifier.isAbstract(c.getModifiers()))
      .filter((Predicate<Class<? extends Converter>>) aClass -> Arrays.stream(aClass.getConstructors())
          .filter(c -> Modifier.isPublic(c.getModifiers()))
          .anyMatch(c -> c.getParameterCount() == 0))
      .collect(Collectors.toSet());
}
 

public Plugin load() {
  ImmutablePlugin.Builder builder = ImmutablePlugin.builder()
      .from(notes(this.pkg))
      .pluginName(AnnotationHelper.pluginName(this.pkg))
      .pluginOwner(AnnotationHelper.pluginOwner(this.pkg));
  List<Plugin.Transformation> transformations = loadTransformations();
  builder.addAllTransformations(transformations);
  List<Plugin.SinkConnector> sinkConnectors = loadSinkConnectors();
  builder.addAllSinkConnectors(sinkConnectors);
  List<Plugin.SourceConnector> sourceConnectors = loadSourceConnectors();
  builder.addAllSourceConnectors(sourceConnectors);
  List<Plugin.Converter> converters = loadConverters();
  builder.addAllConverters(converters);
  return builder.build();
}
 

public KafkaSchemaWrappedSchema(org.apache.pulsar.kafka.shade.avro.Schema schema,
                                Converter converter) {
    Map<String, String> props = new HashMap<>();
    boolean isJsonConverter = converter instanceof JsonConverter;
    props.put(GenericAvroSchema.OFFSET_PROP, isJsonConverter ? "0" : "5");
    this.schemaInfo = SchemaInfo.builder()
            .name(isJsonConverter? "KafKaJson" : "KafkaAvro")
            .type(isJsonConverter ? SchemaType.JSON : SchemaType.AVRO)
            .schema(schema.toString().getBytes(UTF_8))
            .properties(props)
            .build();
}
 

private Connect startConnect(Map<String, String> workerProps) {
    log.info("Scanning for plugin classes. This might take a moment ...");
    Plugins plugins = new Plugins(workerProps);
    // ignore this TCCL switch plugins.compareAndSwapWithDelegatingLoader();
    DistributedConfig config = new DistributedConfig(workerProps);

    String kafkaClusterId = ConnectUtils.lookupKafkaClusterId(config);
    log.debug("Kafka cluster ID: {}", kafkaClusterId);

    RestServer rest = new RestServer(config);
    rest.initializeServer();

    URI advertisedUrl = rest.advertisedUrl();
    String workerId = advertisedUrl.getHost() + ":" + advertisedUrl.getPort();

    KafkaOffsetBackingStore offsetBackingStore = new KafkaOffsetBackingStore();
    offsetBackingStore.configure(config);
    Object connectorClientConfigOverridePolicy = Compatibility.createConnectorClientConfigOverridePolicy(plugins, config);

    Worker worker = Compatibility.createWorker(workerId, time, plugins, config, offsetBackingStore, connectorClientConfigOverridePolicy);
    WorkerConfigTransformer configTransformer = worker.configTransformer();

    Converter internalValueConverter = worker.getInternalValueConverter();
    StatusBackingStore statusBackingStore = new KafkaStatusBackingStore(time, internalValueConverter);
    statusBackingStore.configure(config);

    ConfigBackingStore configBackingStore = new KafkaConfigBackingStore(
        internalValueConverter,
        config,
        configTransformer);

    DistributedHerder herder = Compatibility.createDistributedHerder(config, time, worker,
                                                     kafkaClusterId, statusBackingStore, configBackingStore,
                                                     advertisedUrl.toString(), connectorClientConfigOverridePolicy);

    final Connect connect = new Connect(herder, rest);
    log.info("Kafka Connect distributed worker initialization took {}ms", time.hiResClockMs() - initStart);
    try {
        connect.start();
    } catch (Exception e) {
        log.error("Failed to start Connect", e);
        connect.stop();
        throw new IllegalStateException(e);
    }

    return connect;
}
 

OffsetSetter(Converter internalConverter, KafkaOffsetBackingStore offsetBackingStore) {
  this.internalConverter = internalConverter;
  this.offsetBackingStore = offsetBackingStore;
}
 

/**
 * This method is based on the the standard Kafka Connect start logic in {@link
 * org.apache.kafka.connect.cli.ConnectDistributed#startConnect(Map)}, but with `clientid` prefix
 * support, to prevent JMX metric names from clashing. Also supports command-line property
 * overrides (useful for run-time port configuration), and starts the Mirus {@link
 * HerderStatusMonitor}.
 */
public Connect startConnect(Map<String, String> workerProps) {
  log.info("Scanning for plugin classes. This might take a moment ...");
  Plugins plugins = new Plugins(workerProps);
  plugins.compareAndSwapWithDelegatingLoader();
  DistributedConfig distributedConfig = configWithClientIdSuffix(workerProps, "herder");

  MirusConfig mirusConfig = new MirusConfig(workerProps);

  String kafkaClusterId = ConnectUtils.lookupKafkaClusterId(distributedConfig);
  log.debug("Kafka cluster ID: {}", kafkaClusterId);

  RestServer rest = new RestServer(configWithClientIdSuffix(workerProps, "rest"));
  rest.initializeServer();

  URI advertisedUrl = rest.advertisedUrl();
  String workerId = advertisedUrl.getHost() + ":" + advertisedUrl.getPort();

  KafkaOffsetBackingStore offsetBackingStore = new KafkaOffsetBackingStore();
  offsetBackingStore.configure(configWithClientIdSuffix(workerProps, "offset"));

  WorkerConfig workerConfigs = configWithClientIdSuffix(workerProps, "worker");

  ConnectorClientConfigOverridePolicy connectorClientConfigOverridePolicy =
      plugins.newPlugin(
          distributedConfig.getString(WorkerConfig.CONNECTOR_CLIENT_POLICY_CLASS_CONFIG),
          workerConfigs,
          ConnectorClientConfigOverridePolicy.class);

  Worker worker =
      new Worker(
          workerId,
          time,
          plugins,
          workerConfigs,
          offsetBackingStore,
          connectorClientConfigOverridePolicy);

  WorkerConfigTransformer configTransformer = worker.configTransformer();

  Converter internalValueConverter = worker.getInternalValueConverter();
  StatusBackingStore statusBackingStore =
      new KafkaStatusBackingStore(time, internalValueConverter);
  statusBackingStore.configure(configWithClientIdSuffix(workerProps, "status"));

  ConfigBackingStore configBackingStore =
      new KafkaConfigBackingStore(
          internalValueConverter,
          configWithClientIdSuffix(workerProps, "config"),
          configTransformer);

  DistributedHerder herder =
      new DistributedHerder(
          distributedConfig,
          time,
          worker,
          kafkaClusterId,
          statusBackingStore,
          configBackingStore,
          advertisedUrl.toString(),
          connectorClientConfigOverridePolicy);

  // Initialize HerderStatusMonitor
  boolean autoStartTasks = mirusConfig.getTaskAutoRestart();
  boolean autoStartConnectors = mirusConfig.getConnectorAutoRestart();
  long pollingCycle = mirusConfig.getTaskStatePollingInterval();
  HerderStatusMonitor herderStatusMonitor =
      new HerderStatusMonitor(
          herder, workerId, pollingCycle, autoStartTasks, autoStartConnectors);
  Thread herderStatusMonitorThread = new Thread(herderStatusMonitor);
  herderStatusMonitorThread.setName("herder-status-monitor");

  final Connect connect = new Connect(herder, rest);
  log.info("Mirus worker initialization took {}ms", time.hiResClockMs() - initStart);
  try {
    connect.start();
  } catch (Exception e) {
    log.error("Failed to start Mirus", e);
    connect.stop();
    Exit.exit(3);
  }

  herderStatusMonitorThread.start();

  return connect;
}