org.apache.kafka.common.config.ConfigDef Scala Example

Source File: HiveSinkConfigDef.scala From stream-reactor with Apache License 2.0

5 votes

package com.landoop.streamreactor.connect.hive.sink.config

import java.util

import com.datamountaineer.streamreactor.connect.config.base.traits._
import org.apache.kafka.common.config.ConfigDef
import org.apache.kafka.common.config.ConfigDef.{Importance, Type}

object HiveSinkConfigDef {

  import SinkConfigSettings._

  val config: ConfigDef = new ConfigDef()
    .define(PROGRESS_COUNTER_ENABLED, Type.BOOLEAN, PROGRESS_COUNTER_ENABLED_DEFAULT,
      Importance.MEDIUM, PROGRESS_COUNTER_ENABLED_DOC,
      "Metrics", 1, ConfigDef.Width.MEDIUM, PROGRESS_COUNTER_ENABLED_DISPLAY)
    .define(KcqlKey, Type.STRING, Importance.HIGH, KCQL_DOC)
    .define(DatabaseNameKey, Type.STRING, Importance.HIGH, DatabaseNameDoc)
    .define(MetastoreTypeKey, Type.STRING, Importance.HIGH, MetastoreTypeDoc)
    .define(MetastoreUrisKey, Type.STRING, Importance.HIGH, MetastoreUrisDoc)
    .define(FsDefaultKey, Type.STRING, Importance.HIGH, FsDefaultDoc)

    //config folders
    .define(HdfsConfigDirKey, Type.STRING, HdfsConfigDirDefault, Importance.MEDIUM, HdfsConfigDirDoc, "Configs", 1, ConfigDef.Width.MEDIUM, HdfsConfigDirDisplay)
    .define(HiveConfigDirKey, Type.STRING, HiveConfigDirDefault, Importance.MEDIUM, HiveConfigDirDoc, "Configs", 2, ConfigDef.Width.MEDIUM, HiveConfigDirDisplay)

    //security
    .define(KerberosKey, Type.BOOLEAN, KerberosDefault, Importance.MEDIUM, KerberosDoc, "Security", 1, ConfigDef.Width.MEDIUM, KerberosDisplay)
    .define(KerberosAuthModeKey, Type.STRING, KerberosAuthModeDefault, Importance.MEDIUM, KerberosAuthModeDoc, "Security", 2, ConfigDef.Width.MEDIUM, KerberosAuthModeDisplay)


    //keytab
    .define(PrincipalKey, Type.STRING, PrincipalDefault, Importance.MEDIUM, PrincipalDoc, "Kerberos Keytab", 1, ConfigDef.Width.MEDIUM, PrincipalDisplay)
    .define(KerberosKeyTabKey, Type.STRING, KerberosKeyTabDefault, Importance.MEDIUM, KerberosKeyTabDoc, "Kerberos Keytab", 2, ConfigDef.Width.MEDIUM, KerberosKeyTabDisplay)
    .define(NameNodePrincipalKey, Type.STRING, NameNodePrincipalDefault, Importance.MEDIUM, NameNodePrincipalDoc, "Kerberos Keytab", 3, ConfigDef.Width.MEDIUM, NameNodePrincipalDisplay)
    .define(KerberosTicketRenewalKey, Type.LONG, KerberosTicketRenewalDefault, Importance.MEDIUM, KerberosTicketRenewalDoc, "Kerberos Keytab", 4, ConfigDef.Width.MEDIUM, KerberosTicketRenewalDisplay)


    //user password
    .define(KerberosUserKey, Type.STRING, KerberosUserDefault, Importance.MEDIUM, KerberosUserDoc, "Kerberos User Password", 1, ConfigDef.Width.MEDIUM, KerberosUserDisplay)
    .define(KerberosPasswordKey, Type.PASSWORD, KerberosPasswordDefault, Importance.MEDIUM, KerberosPasswordDoc, "Kerberos User Password", 2, ConfigDef.Width.MEDIUM, KerberosPasswordDisplay)
    .define(KerberosKrb5Key, Type.STRING, KerberosKrb5Default, Importance.MEDIUM, KerberosKrb5Doc, "Kerberos User Password", 3, ConfigDef.Width.MEDIUM, KerberosKrb5Display)
    .define(KerberosJaasKey, Type.STRING, KerberosJaasDefault, Importance.MEDIUM, KerberosJaasDoc, "Kerberos User Password", 4, ConfigDef.Width.MEDIUM, KerberosJaasDisplay)
    .define(JaasEntryNameKey, Type.STRING, JaasEntryNameDefault, Importance.MEDIUM, JaasEntryNameDoc, "Kerberos User Password", 5, ConfigDef.Width.MEDIUM, JaasEntryNameDisplay)
}

case class HiveSinkConfigDefBuilder(props: util.Map[String, String])
  extends BaseConfig(SinkConfigSettings.CONNECTOR_PREFIX, HiveSinkConfigDef.config, props)
    with KcqlSettings
    with ErrorPolicySettings
    with NumberRetriesSettings
    with UserSettings
    with ConnectionSettings

Source File: ReThinkSinkConfig.scala From stream-reactor with Apache License 2.0

5 votes

package com.datamountaineer.streamreactor.connect.rethink.config

import java.util

import com.datamountaineer.streamreactor.connect.config.base.traits._
import org.apache.kafka.common.config.ConfigDef
import org.apache.kafka.common.config.ConfigDef.{Importance, Type}


object ReThinkSinkConfig extends ReThinkConfig {

  val config: ConfigDef = baseConfig
    .define(ReThinkConfigConstants.ERROR_POLICY, Type.STRING,
      ReThinkConfigConstants.ERROR_POLICY_DEFAULT,
      Importance.HIGH, ReThinkConfigConstants.ERROR_POLICY_DOC,
      "Connection", 9, ConfigDef.Width.MEDIUM, ReThinkConfigConstants.ERROR_POLICY)

    .define(ReThinkConfigConstants.ERROR_RETRY_INTERVAL, Type.INT,
      ReThinkConfigConstants.ERROR_RETRY_INTERVAL_DEFAULT,
      Importance.MEDIUM, ReThinkConfigConstants.ERROR_RETRY_INTERVAL_DOC,
      "Connection", 10, ConfigDef.Width.MEDIUM, ReThinkConfigConstants.ERROR_RETRY_INTERVAL)

    .define(ReThinkConfigConstants.NBR_OF_RETRIES, Type.INT,
      ReThinkConfigConstants.NBR_OF_RETIRES_DEFAULT,
      Importance.MEDIUM, ReThinkConfigConstants.NBR_OF_RETRIES_DOC,
      "Connection", 11, ConfigDef.Width.MEDIUM, ReThinkConfigConstants.NBR_OF_RETRIES)
}

case class ReThinkSinkConfig(props: util.Map[String, String])
  extends BaseConfig(ReThinkConfigConstants.RETHINK_CONNECTOR_PREFIX, ReThinkSinkConfig.config, props)
    with ErrorPolicySettings
    with NumberRetriesSettings
    with KcqlSettings
    with DatabaseSettings
    with RetryIntervalSettings

Source File: ReThinkConfig.scala From stream-reactor with Apache License 2.0

5 votes

package com.datamountaineer.streamreactor.connect.rethink.config

import org.apache.kafka.common.config.ConfigDef
import org.apache.kafka.common.config.ConfigDef.{Importance, Type}


trait ReThinkConfig {
  val baseConfig: ConfigDef = new ConfigDef()
    .define(ReThinkConfigConstants.RETHINK_HOST, Type.STRING,
      ReThinkConfigConstants.RETHINK_HOST_DEFAULT,
      Importance.HIGH, ReThinkConfigConstants.RETHINK_HOST_DOC,
      "Connection", 1, ConfigDef.Width.MEDIUM, ReThinkConfigConstants.RETHINK_HOST)

    .define(ReThinkConfigConstants.RETHINK_DB, Type.STRING,
      ReThinkConfigConstants.RETHINK_DB_DEFAULT,
      Importance.HIGH, ReThinkConfigConstants.RETHINK_DB_DOC,
      "Connection", 2, ConfigDef.Width.MEDIUM, ReThinkConfigConstants.RETHINK_DB)

    .define(ReThinkConfigConstants.RETHINK_PORT, Type.INT,
      ReThinkConfigConstants.RETHINK_PORT_DEFAULT,
      Importance.MEDIUM, ReThinkConfigConstants.RETHINK_PORT_DOC,
      "Connection", 3, ConfigDef.Width.MEDIUM, ReThinkConfigConstants.RETHINK_PORT)

    .define(ReThinkConfigConstants.KCQL, Type.STRING, Importance.HIGH,
      ReThinkConfigConstants.KCQL_DOC,
      "Connection", 4, ConfigDef.Width.MEDIUM, ReThinkConfigConstants.KCQL)

    .define(ReThinkConfigConstants.USERNAME, Type.STRING,
      ReThinkConfigConstants.USERNAME_DEFAULT,
      Importance.MEDIUM, ReThinkConfigConstants.USERNAME_DOC,
      "Connection", 5, ConfigDef.Width.MEDIUM, ReThinkConfigConstants.USERNAME)

    .define(ReThinkConfigConstants.PASSWORD, Type.PASSWORD,
      ReThinkConfigConstants.PASSWORD_DEFAULT,
      Importance.MEDIUM, ReThinkConfigConstants.PASSWORD_DOC,
      "Connection", 6, ConfigDef.Width.MEDIUM, ReThinkConfigConstants.PASSWORD)

    .define(ReThinkConfigConstants.AUTH_KEY, Type.PASSWORD,
      ReThinkConfigConstants.AUTH_KEY_DEFAULT,
      Importance.MEDIUM, ReThinkConfigConstants.AUTH_KEY_DOC,
      "Connection", 7, ConfigDef.Width.MEDIUM, ReThinkConfigConstants.AUTH_KEY)

    .define(ReThinkConfigConstants.CERT_FILE, Type.STRING,
      ReThinkConfigConstants.CERT_FILE_DEFAULT,
      Importance.MEDIUM, ReThinkConfigConstants.CERT_FILE_DOC,
      "Connection", 8, ConfigDef.Width.MEDIUM, ReThinkConfigConstants.CERT_FILE)

    .define(ReThinkConfigConstants.PROGRESS_COUNTER_ENABLED, Type.BOOLEAN,
      ReThinkConfigConstants.PROGRESS_COUNTER_ENABLED_DEFAULT, Importance.MEDIUM,
      ReThinkConfigConstants.PROGRESS_COUNTER_ENABLED_DOC,
      "Metrics", 1, ConfigDef.Width.MEDIUM, ReThinkConfigConstants.PROGRESS_COUNTER_ENABLED_DISPLAY)
}

Source File: KuduConfig.scala From stream-reactor with Apache License 2.0

5 votes

package com.datamountaineer.streamreactor.connect.kudu.config

import java.util

import com.datamountaineer.streamreactor.connect.config.base.traits._
import org.apache.kafka.common.config.ConfigDef
import org.apache.kafka.common.config.ConfigDef.{Importance, Type}



object KuduConfig {


  val config: ConfigDef = new ConfigDef()
    .define(KuduConfigConstants.KUDU_MASTER, Type.STRING, KuduConfigConstants.KUDU_MASTER_DEFAULT,
      Importance.HIGH, KuduConfigConstants.KUDU_MASTER_DOC,
      "Connection", 1, ConfigDef.Width.MEDIUM, KuduConfigConstants.KUDU_MASTER)

    .define(KuduConfigConstants.KCQL, Type.STRING, Importance.HIGH, KuduConfigConstants.KCQL,
      "Connection", 2, ConfigDef.Width.MEDIUM, KuduConfigConstants.KCQL)

    .define(KuduConfigConstants.ERROR_POLICY, Type.STRING, KuduConfigConstants.ERROR_POLICY_DEFAULT,
      Importance.HIGH, KuduConfigConstants.ERROR_POLICY_DOC,
      "Connection", 3, ConfigDef.Width.MEDIUM, KuduConfigConstants.ERROR_POLICY)

    .define(KuduConfigConstants.ERROR_RETRY_INTERVAL, Type.INT, KuduConfigConstants.ERROR_RETRY_INTERVAL_DEFAULT,
      Importance.MEDIUM, KuduConfigConstants.ERROR_RETRY_INTERVAL_DOC,
      "Connection", 4, ConfigDef.Width.MEDIUM, KuduConfigConstants.ERROR_RETRY_INTERVAL)

    .define(KuduConfigConstants.NBR_OF_RETRIES, Type.INT, KuduConfigConstants.NBR_OF_RETIRES_DEFAULT,
      Importance.MEDIUM, KuduConfigConstants.NBR_OF_RETRIES_DOC,
      "Connection", 5, ConfigDef.Width.MEDIUM, KuduConfigConstants.NBR_OF_RETRIES)

    .define(KuduConfigConstants.SCHEMA_REGISTRY_URL, Type.STRING,
      KuduConfigConstants.SCHEMA_REGISTRY_URL_DEFAULT, Importance.HIGH, KuduConfigConstants.SCHEMA_REGISTRY_URL_DOC,
      "Connection", 6, ConfigDef.Width.MEDIUM,
      KuduConfigConstants.SCHEMA_REGISTRY_URL)

    .define(KuduConfigConstants.WRITE_FLUSH_MODE, Type.STRING, KuduConfigConstants.WRITE_FLUSH_MODE_DEFAULT,
      Importance.MEDIUM, KuduConfigConstants.WRITE_FLUSH_MODE_DOC,
      "Connection", 7, ConfigDef.Width.MEDIUM,
      KuduConfigConstants.WRITE_FLUSH_MODE)

    .define(KuduConfigConstants.MUTATION_BUFFER_SPACE, Type.INT, KuduConfigConstants.MUTATION_BUFFER_SPACE_DEFAULT,
      Importance.MEDIUM, KuduConfigConstants.MUTATION_BUFFER_SPACE_DOC,
      "Connection", 8, ConfigDef.Width.MEDIUM,
      KuduConfigConstants.MUTATION_BUFFER_SPACE)

    .define(KuduConfigConstants.PROGRESS_COUNTER_ENABLED, Type.BOOLEAN, KuduConfigConstants.PROGRESS_COUNTER_ENABLED_DEFAULT,
      Importance.MEDIUM, KuduConfigConstants.PROGRESS_COUNTER_ENABLED_DOC,
      "Metrics", 1, ConfigDef.Width.MEDIUM, KuduConfigConstants.PROGRESS_COUNTER_ENABLED_DISPLAY)


}

class KuduConfig(props: util.Map[String, String])
  extends BaseConfig(KuduConfigConstants.CONNECTOR_PREFIX, KuduConfig.config, props)
    with KcqlSettings
    with DatabaseSettings
    with ErrorPolicySettings
    with NumberRetriesSettings
    with ConnectionSettings {

  def getWriteFlushMode() = WriteFlushMode.withName(
    props.getOrDefault(
      KuduConfigConstants.WRITE_FLUSH_MODE,
      KuduConfigConstants.WRITE_FLUSH_MODE_DEFAULT).toUpperCase)

}

Source File: ElasticSinkConnector.scala From stream-reactor with Apache License 2.0

5 votes

package com.datamountaineer.streamreactor.connect.elastic6

import java.util

import com.datamountaineer.streamreactor.connect.config.Helpers
import com.datamountaineer.streamreactor.connect.elastic6.config.{ElasticConfig, ElasticConfigConstants}
import com.datamountaineer.streamreactor.connect.utils.JarManifest
import com.typesafe.scalalogging.StrictLogging
import org.apache.kafka.common.config.ConfigDef
import org.apache.kafka.connect.connector.Task
import org.apache.kafka.connect.sink.SinkConnector

import scala.collection.JavaConverters._

class ElasticSinkConnector extends SinkConnector with StrictLogging {
  private var configProps : Option[util.Map[String, String]] = None
  private val configDef = ElasticConfig.config
  private val manifest = JarManifest(getClass.getProtectionDomain.getCodeSource.getLocation)

  
  override def start(props: util.Map[String, String]): Unit = {
    logger.info(s"Starting Elastic sink task.")
    Helpers.checkInputTopics(ElasticConfigConstants.KCQL, props.asScala.toMap)
    configProps = Some(props)
  }

  override def stop(): Unit = {}
  override def version(): String = manifest.version()
  override def config(): ConfigDef = configDef
}

Source File: PulsarSinkConnector.scala From stream-reactor with Apache License 2.0

5 votes

package com.datamountaineer.streamreactor.connect.pulsar.sink

import java.util

import com.datamountaineer.streamreactor.connect.config.Helpers
import com.datamountaineer.streamreactor.connect.pulsar.config.{PulsarConfigConstants, PulsarSinkConfig}
import com.datamountaineer.streamreactor.connect.utils.JarManifest
import com.typesafe.scalalogging.StrictLogging
import org.apache.kafka.common.config.ConfigDef
import org.apache.kafka.connect.connector.Task
import org.apache.kafka.connect.sink.SinkConnector

import scala.collection.JavaConverters._

class PulsarSinkConnector extends SinkConnector with StrictLogging {
  private val configDef = PulsarSinkConfig.config
  private var configProps: Option[util.Map[String, String]] = None
  private val manifest = JarManifest(getClass.getProtectionDomain.getCodeSource.getLocation)

  override def start(props: util.Map[String, String]): Unit = {
    logger.info(s"Starting Pulsar sink connector.")
    Helpers.checkInputTopics(PulsarConfigConstants.KCQL_CONFIG, props.asScala.toMap)
    configProps = Some(props)
  }

  override def taskClass(): Class[_ <: Task] = classOf[PulsarSinkTask]

  override def version(): String = manifest.version()

  override def stop(): Unit = {}

  override def taskConfigs(maxTasks: Int): util.List[util.Map[String, String]] = {
    logger.info(s"Setting task configurations for $maxTasks workers.")
    (1 to maxTasks).map(_ => configProps.get).toList.asJava
  }

  override def config(): ConfigDef = configDef
}

Source File: PulsarConfig.scala From stream-reactor with Apache License 2.0

5 votes

package com.datamountaineer.streamreactor.connect.pulsar.config

import java.util

import com.datamountaineer.streamreactor.connect.config.base.traits._
import org.apache.kafka.common.config.ConfigDef
import org.apache.kafka.common.config.ConfigDef.{Importance, Type}

object PulsarConfig {
  val config: ConfigDef = new ConfigDef()
    .define(PulsarConfigConstants.HOSTS_CONFIG, Type.STRING, Importance.HIGH, PulsarConfigConstants.HOSTS_DOC,
      "Connection", 1, ConfigDef.Width.MEDIUM, PulsarConfigConstants.HOSTS_DISPLAY)

    //ssl
    .define(PulsarConfigConstants.SSL_CA_CERT_CONFIG, Type.STRING, null, Importance.MEDIUM, PulsarConfigConstants.SSL_CA_CERT_DOC,
      "TLS", 1, ConfigDef.Width.MEDIUM, PulsarConfigConstants.SSL_CA_CERT_DISPLAY)
    .define(PulsarConfigConstants.SSL_CERT_CONFIG, Type.STRING, null, Importance.MEDIUM, PulsarConfigConstants.SSL_CERT_DOC,
      "TLS", 2, ConfigDef.Width.MEDIUM, PulsarConfigConstants.SSL_CERT_DISPLAY)
    .define(PulsarConfigConstants.SSL_CERT_KEY_CONFIG, Type.STRING, null, Importance.MEDIUM, PulsarConfigConstants.SSL_CERT_KEY_DOC,
      "TLS", 3, ConfigDef.Width.MEDIUM, PulsarConfigConstants.SSL_CERT_KEY_DISPLAY)

    //kcql
    .define(PulsarConfigConstants.KCQL_CONFIG, Type.STRING, Importance.HIGH, PulsarConfigConstants.KCQL_DOC,
      "KCQL", 1, ConfigDef.Width.MEDIUM, PulsarConfigConstants.KCQL_DISPLAY)

    .define(PulsarConfigConstants.PROGRESS_COUNTER_ENABLED, Type.BOOLEAN, PulsarConfigConstants.PROGRESS_COUNTER_ENABLED_DEFAULT,
      Importance.MEDIUM, PulsarConfigConstants.PROGRESS_COUNTER_ENABLED_DOC,
      "Metrics", 1, ConfigDef.Width.MEDIUM, PulsarConfigConstants.PROGRESS_COUNTER_ENABLED_DISPLAY)
}

object PulsarSourceConfig {
  val config = PulsarConfig.config
    //converter
    .define(PulsarConfigConstants.THROW_ON_CONVERT_ERRORS_CONFIG, Type.BOOLEAN, PulsarConfigConstants.THROW_ON_CONVERT_ERRORS_DEFAULT,
      Importance.HIGH, PulsarConfigConstants.THROW_ON_CONVERT_ERRORS_DOC,
      "Converter", 1, ConfigDef.Width.MEDIUM, PulsarConfigConstants.THROW_ON_CONVERT_ERRORS_DISPLAY)
    .define(PulsarConfigConstants.AVRO_CONVERTERS_SCHEMA_FILES, Type.STRING, PulsarConfigConstants.AVRO_CONVERTERS_SCHEMA_FILES_DEFAULT,
      Importance.HIGH, PulsarConfigConstants.AVRO_CONVERTERS_SCHEMA_FILES_DOC, "Converter", 3, ConfigDef.Width.MEDIUM,
      PulsarConfigConstants.AVRO_CONVERTERS_SCHEMA_FILES)
    //manager
    .define(PulsarConfigConstants.POLLING_TIMEOUT_CONFIG, Type.INT, PulsarConfigConstants.POLLING_TIMEOUT_DEFAULT,
      Importance.LOW, PulsarConfigConstants.POLLING_TIMEOUT_DOC,
      "Manager", 1, ConfigDef.Width.MEDIUM, PulsarConfigConstants.POLLING_TIMEOUT_DISPLAY)
    .define(PulsarConfigConstants.INTERNAL_BATCH_SIZE, Type.INT, PulsarConfigConstants.INTERNAL_BATCH_SIZE_DEFAULT,
      Importance.LOW, PulsarConfigConstants.INTERNAL_BATCH_SIZE_DOC,
      "Manager", 2, ConfigDef.Width.MEDIUM, PulsarConfigConstants.INTERNAL_BATCH_SIZE)

}

case class PulsarSourceConfig(props: util.Map[String, String])
  extends BaseConfig(PulsarConfigConstants.CONNECTOR_PREFIX, PulsarSourceConfig.config, props) with PulsarConfigBase

object PulsarSinkConfig {
  val config = PulsarConfig.config

    .define(PulsarConfigConstants.ERROR_POLICY, Type.STRING, PulsarConfigConstants.ERROR_POLICY_DEFAULT,
      Importance.HIGH, PulsarConfigConstants.ERROR_POLICY_DOC,
      "Connection", 9, ConfigDef.Width.MEDIUM, PulsarConfigConstants.ERROR_POLICY)

    .define(PulsarConfigConstants.ERROR_RETRY_INTERVAL, Type.INT, PulsarConfigConstants.ERROR_RETRY_INTERVAL_DEFAULT,
      Importance.MEDIUM, PulsarConfigConstants.ERROR_RETRY_INTERVAL_DOC,
      "Connection", 10, ConfigDef.Width.MEDIUM, PulsarConfigConstants.ERROR_RETRY_INTERVAL)

    .define(PulsarConfigConstants.NBR_OF_RETRIES, Type.INT, PulsarConfigConstants.NBR_OF_RETIRES_DEFAULT,
      Importance.MEDIUM, PulsarConfigConstants.NBR_OF_RETRIES_DOC,
      "Connection", 11, ConfigDef.Width.MEDIUM, PulsarConfigConstants.NBR_OF_RETRIES)
}

case class PulsarSinkConfig(props: util.Map[String, String])
  extends BaseConfig(PulsarConfigConstants.CONNECTOR_PREFIX, PulsarSinkConfig.config, props) with PulsarConfigBase

case class PulsarConfig(props: util.Map[String, String])
  extends BaseConfig(PulsarConfigConstants.CONNECTOR_PREFIX, PulsarConfig.config, props) with PulsarConfigBase

trait PulsarConfigBase extends KcqlSettings
  with NumberRetriesSettings
  with ErrorPolicySettings
  with SSLSettings
  with ConnectionSettings
  with UserSettings

Source File: RedisConfig.scala From stream-reactor with Apache License 2.0

5 votes

package com.datamountaineer.streamreactor.connect.redis.sink.config

import java.util

import com.datamountaineer.streamreactor.connect.config.base.traits._
import org.apache.kafka.common.config.ConfigDef
import org.apache.kafka.common.config.ConfigDef.{Importance, Type}

object RedisConfig {

  val config: ConfigDef = new ConfigDef()
    .define(RedisConfigConstants.REDIS_HOST, Type.STRING, Importance.HIGH, RedisConfigConstants.REDIS_HOST_DOC,
      "Connection", 2, ConfigDef.Width.MEDIUM, RedisConfigConstants.REDIS_HOST)
    .define(RedisConfigConstants.REDIS_PORT, Type.INT, Importance.HIGH, RedisConfigConstants.REDIS_PORT_DOC,
      "Connection", 3, ConfigDef.Width.MEDIUM, RedisConfigConstants.REDIS_PORT)
    .define(RedisConfigConstants.REDIS_PASSWORD, Type.PASSWORD, null, Importance.LOW, RedisConfigConstants.REDIS_PASSWORD_DOC,
      "Connection", 4, ConfigDef.Width.MEDIUM, RedisConfigConstants.REDIS_PASSWORD)
    .define(RedisConfigConstants.REDIS_SSL_ENABLED, Type.BOOLEAN, false, Importance.LOW, RedisConfigConstants.REDIS_SSL_ENABLED_DOC,
        "Connection", 5, ConfigDef.Width.MEDIUM, RedisConfigConstants.REDIS_SSL_ENABLED)
    .define(RedisConfigConstants.KCQL_CONFIG, Type.STRING, Importance.HIGH, RedisConfigConstants.KCQL_DOC,
      "Connection", 1, ConfigDef.Width.MEDIUM, RedisConfigConstants.KCQL_CONFIG)
    .define(RedisConfigConstants.ERROR_POLICY, Type.STRING, RedisConfigConstants.ERROR_POLICY_DEFAULT,
      Importance.HIGH, RedisConfigConstants.ERROR_POLICY_DOC,
      "Connection", 5, ConfigDef.Width.MEDIUM, RedisConfigConstants.ERROR_POLICY)
    .define(RedisConfigConstants.ERROR_RETRY_INTERVAL, Type.INT, RedisConfigConstants.ERROR_RETRY_INTERVAL_DEFAULT,
      Importance.MEDIUM, RedisConfigConstants.ERROR_RETRY_INTERVAL_DOC,
      "Connection", 6, ConfigDef.Width.MEDIUM, RedisConfigConstants.ERROR_RETRY_INTERVAL)
    .define(RedisConfigConstants.NBR_OF_RETRIES, Type.INT, RedisConfigConstants.NBR_OF_RETIRES_DEFAULT,
      Importance.MEDIUM, RedisConfigConstants.NBR_OF_RETRIES_DOC,
      "Connection", 7, ConfigDef.Width.MEDIUM, RedisConfigConstants.NBR_OF_RETRIES)
    .define(RedisConfigConstants.PROGRESS_COUNTER_ENABLED, Type.BOOLEAN, RedisConfigConstants.PROGRESS_COUNTER_ENABLED_DEFAULT,
      Importance.MEDIUM, RedisConfigConstants.PROGRESS_COUNTER_ENABLED_DOC,
      "Metrics", 1, ConfigDef.Width.MEDIUM, RedisConfigConstants.PROGRESS_COUNTER_ENABLED_DISPLAY)
    .define(RedisConfigConstants.REDIS_PK_DELIMITER, Type.STRING,
      RedisConfigConstants.REDIS_PK_DELIMITER_DEFAULT_VALUE,
      Importance.LOW, RedisConfigConstants.REDIS_PK_DELIMITER_DOC)
    .withClientSslSupport()
}


case class RedisConfig(props: util.Map[String, String])
  extends BaseConfig(RedisConfigConstants.CONNECTOR_PREFIX, RedisConfig.config, props)
    with KcqlSettings
    with ErrorPolicySettings
    with NumberRetriesSettings
    with UserSettings

Source File: MqttSinkConnector.scala From stream-reactor with Apache License 2.0

5 votes

package com.datamountaineer.streamreactor.connect.mqtt.sink

import java.util

import com.datamountaineer.streamreactor.connect.config.Helpers
import com.datamountaineer.streamreactor.connect.mqtt.config.{MqttConfigConstants, MqttSinkConfig}
import com.datamountaineer.streamreactor.connect.utils.JarManifest
import com.typesafe.scalalogging.StrictLogging
import org.apache.kafka.common.config.ConfigDef
import org.apache.kafka.connect.connector.Task
import org.apache.kafka.connect.sink.SinkConnector

import scala.collection.JavaConverters._


class MqttSinkConnector extends SinkConnector with StrictLogging {
  private val configDef = MqttSinkConfig.config
  private var configProps: Option[util.Map[String, String]] = None
  private val manifest = JarManifest(getClass.getProtectionDomain.getCodeSource.getLocation)

  override def start(props: util.Map[String, String]): Unit = {
    logger.info(s"Starting Mqtt sink connector.")
    Helpers.checkInputTopics(MqttConfigConstants.KCQL_CONFIG, props.asScala.toMap)
    configProps = Some(props)
  }

  override def taskClass(): Class[_ <: Task] = classOf[MqttSinkTask]

  override def version(): String = manifest.version()

  override def stop(): Unit = {}

  override def taskConfigs(maxTasks: Int): util.List[util.Map[String, String]] = {
    logger.info(s"Setting task configurations for $maxTasks workers.")
    (1 to maxTasks).map(_ => configProps.get).toList.asJava
  }

  override def config(): ConfigDef = configDef
}

Source File: HiveSinkConnector.scala From stream-reactor with Apache License 2.0

5 votes

package com.landoop.streamreactor.connect.hive.sink

import java.util

import com.datamountaineer.streamreactor.connect.utils.JarManifest
import com.landoop.streamreactor.connect.hive.sink.config.HiveSinkConfigDef
import org.apache.kafka.common.config.ConfigDef
import org.apache.kafka.connect.connector.Task
import org.apache.kafka.connect.sink.SinkConnector

import scala.collection.JavaConverters._

class HiveSinkConnector extends SinkConnector {

  val logger = org.slf4j.LoggerFactory.getLogger(getClass.getName)

  private val manifest = JarManifest(getClass.getProtectionDomain.getCodeSource.getLocation)
  private var props: util.Map[String, String] = _

  override def version(): String = manifest.version()
  override def taskClass(): Class[_ <: Task] = classOf[HiveSinkTask]
  override def config(): ConfigDef = HiveSinkConfigDef.config

  override def start(props: util.Map[String, String]): Unit = {
    logger.info(s"Creating hive sink connector")
    this.props = props
  }

  override def stop(): Unit = ()

  override def taskConfigs(maxTasks: Int): util.List[util.Map[String, String]] = {
    logger.info(s"Creating $maxTasks tasks config")
    List.fill(maxTasks)(props).asJava
  }
}

Source File: ReThinkSourceConfig.scala From stream-reactor with Apache License 2.0

5 votes

package com.datamountaineer.streamreactor.connect.rethink.config

import java.util

import com.datamountaineer.streamreactor.connect.config.base.traits.{BaseConfig, BatchSizeSettings, DatabaseSettings, KcqlSettings}
import org.apache.kafka.common.config.ConfigDef
import org.apache.kafka.common.config.ConfigDef.{Importance, Type}


object ReThinkSourceConfig extends ReThinkConfig {
  val config: ConfigDef = baseConfig

    .define(ReThinkConfigConstants.SOURCE_LINGER_MS, Type.LONG,
      ReThinkConfigConstants.SOURCE_LINGER_MS_DEFAULT,
      Importance.MEDIUM, ReThinkConfigConstants.SOURCE_LINGER_MS_DOC,
      "Connection", 10, ConfigDef.Width.MEDIUM, ReThinkConfigConstants.SOURCE_LINGER_MS)
}

case class ReThinkSourceConfig(props: util.Map[String, String])
  extends BaseConfig(ReThinkConfigConstants.RETHINK_CONNECTOR_PREFIX, ReThinkSourceConfig.config, props)
    with KcqlSettings
    with DatabaseSettings
    with BatchSizeSettings

Source File: HiveSourceConnector.scala From stream-reactor with Apache License 2.0

5 votes

package com.landoop.streamreactor.connect.hive.source

import java.util

import com.datamountaineer.streamreactor.connect.utils.JarManifest
import com.landoop.streamreactor.connect.hive.sink.config.HiveSinkConfigDef
import com.typesafe.scalalogging.StrictLogging
import org.apache.kafka.common.config.ConfigDef
import org.apache.kafka.connect.connector.Task
import org.apache.kafka.connect.source.SourceConnector

import scala.collection.JavaConverters._

class HiveSourceConnector extends SourceConnector with StrictLogging {

  private val manifest = JarManifest(getClass.getProtectionDomain.getCodeSource.getLocation)
  private var props: util.Map[String, String] = _

  override def version(): String = manifest.version()
  override def taskClass(): Class[_ <: Task] = classOf[HiveSourceTask]
  override def config(): ConfigDef = HiveSinkConfigDef.config

  override def start(props: util.Map[String, String]): Unit = {
    logger.info(s"Creating hive sink connector")
    this.props = props
  }

  override def stop(): Unit = ()

  override def taskConfigs(maxTasks: Int): util.List[util.Map[String, String]] = {
    logger.info(s"Creating $maxTasks tasks config")
    List.fill(maxTasks)(props).asJava
  }
}

Source File: HiveSourceConfigDef.scala From stream-reactor with Apache License 2.0

5 votes

package com.landoop.streamreactor.connect.hive.source.config

import java.util

import com.datamountaineer.streamreactor.connect.config.base.traits._
import org.apache.kafka.common.config.ConfigDef
import org.apache.kafka.common.config.ConfigDef.{Importance, Type}

object HiveSourceConfigDef {

  import HiveSourceConfigConstants._

  val config: ConfigDef = new ConfigDef()
    .define(PROGRESS_COUNTER_ENABLED, Type.BOOLEAN, PROGRESS_COUNTER_ENABLED_DEFAULT,
      Importance.MEDIUM, PROGRESS_COUNTER_ENABLED_DOC,
      "Metrics", 1, ConfigDef.Width.MEDIUM, PROGRESS_COUNTER_ENABLED_DISPLAY)
    .define(KcqlKey, Type.STRING, Importance.HIGH, KCQL_DOC)
    .define(DatabaseNameKey, Type.STRING, Importance.HIGH, DatabaseNameDoc)
    .define(MetastoreTypeKey, Type.STRING, Importance.HIGH, MetastoreTypeDoc)
    .define(MetastoreUrisKey, Type.STRING, Importance.HIGH, MetastoreUrisDoc)
    .define(FsDefaultKey, Type.STRING, Importance.HIGH, FsDefaultDoc)
    .define(PollSizeKey, Type.INT, 1024, Importance.HIGH, PollSizeDoc)

    //config folders
    .define(HdfsConfigDirKey, Type.STRING, HdfsConfigDirDefault, Importance.MEDIUM, HdfsConfigDirDoc, "Configs", 1, ConfigDef.Width.MEDIUM, HdfsConfigDirDisplay)
    .define(HiveConfigDirKey, Type.STRING, HiveConfigDirDefault, Importance.MEDIUM, HiveConfigDirDoc, "Configs", 2, ConfigDef.Width.MEDIUM, HiveConfigDirDisplay)

    //security
    .define(KerberosKey, Type.BOOLEAN, KerberosDefault, Importance.MEDIUM, KerberosDoc, "Security", 1, ConfigDef.Width.MEDIUM, KerberosDisplay)
    .define(KerberosAuthModeKey, Type.STRING, KerberosAuthModeDefault, Importance.MEDIUM, KerberosAuthModeDoc, "Security", 2, ConfigDef.Width.MEDIUM, KerberosAuthModeDisplay)


    //keytab
    .define(PrincipalKey, Type.STRING, PrincipalDefault, Importance.MEDIUM, PrincipalDoc, "Kerberos Keytab", 1, ConfigDef.Width.MEDIUM, PrincipalDisplay)
    .define(KerberosKeyTabKey, Type.STRING, KerberosKeyTabDefault, Importance.MEDIUM, KerberosKeyTabDoc, "Kerberos Keytab", 2, ConfigDef.Width.MEDIUM, KerberosKeyTabDisplay)
    .define(NameNodePrincipalKey, Type.STRING, NameNodePrincipalDefault, Importance.MEDIUM, NameNodePrincipalDoc, "Kerberos Keytab", 3, ConfigDef.Width.MEDIUM, NameNodePrincipalDisplay)
    .define(KerberosTicketRenewalKey, Type.LONG, KerberosTicketRenewalDefault, Importance.MEDIUM, KerberosTicketRenewalDoc, "Kerberos Keytab", 4, ConfigDef.Width.MEDIUM, KerberosTicketRenewalDisplay)


    //user password
    .define(KerberosUserKey, Type.STRING, KerberosUserDefault, Importance.MEDIUM, KerberosUserDoc, "Kerberos User Password", 1, ConfigDef.Width.MEDIUM, KerberosUserDisplay)
    .define(KerberosPasswordKey, Type.PASSWORD, KerberosPasswordDefault, Importance.MEDIUM, KerberosPasswordDoc, "Kerberos User Password", 2, ConfigDef.Width.MEDIUM, KerberosPasswordDisplay)
    .define(KerberosKrb5Key, Type.STRING, KerberosKrb5Default, Importance.MEDIUM, KerberosKrb5Doc, "Kerberos User Password", 3, ConfigDef.Width.MEDIUM, KerberosKrb5Display)
    .define(KerberosJaasKey, Type.STRING, KerberosJaasDefault, Importance.MEDIUM, KerberosJaasDoc, "Kerberos User Password", 4, ConfigDef.Width.MEDIUM, KerberosJaasDisplay)
    .define(JaasEntryNameKey, Type.STRING, JaasEntryNameDefault, Importance.MEDIUM, JaasEntryNameDoc, "Kerberos User Password", 5, ConfigDef.Width.MEDIUM, JaasEntryNameDisplay)

}

case class HiveSourceConfigDefBuilder(props: util.Map[String, String])
  extends BaseConfig(HiveSourceConfigConstants.CONNECTOR_PREFIX, HiveSourceConfigDef.config, props)
    with KcqlSettings
    with ErrorPolicySettings
    with NumberRetriesSettings
    with UserSettings
    with ConnectionSettings

Source File: VoltSinkConfig.scala From stream-reactor with Apache License 2.0

5 votes

package com.datamountaineer.streamreactor.connect.voltdb.config

import java.util

import com.datamountaineer.streamreactor.connect.config.base.traits._
import org.apache.kafka.common.config.ConfigDef
import org.apache.kafka.common.config.ConfigDef.{Importance, Type}

object VoltSinkConfig {
  val config: ConfigDef = new ConfigDef()
    .define(VoltSinkConfigConstants.SERVERS_CONFIG, Type.STRING, Importance.HIGH, VoltSinkConfigConstants.SERVERS_DOC,
      "Connection", 1, ConfigDef.Width.MEDIUM, VoltSinkConfigConstants.SERVERS_CONFIG)
    .define(VoltSinkConfigConstants.USER_CONFIG, Type.STRING, Importance.HIGH, VoltSinkConfigConstants.USER_DOC,
      "Connection", 2, ConfigDef.Width.MEDIUM, VoltSinkConfigConstants.USER_CONFIG)
    .define(VoltSinkConfigConstants.PASSWORD_CONFIG, Type.PASSWORD, Importance.HIGH, VoltSinkConfigConstants.PASSWORD_DOC,
      "Connection", 3, ConfigDef.Width.MEDIUM, VoltSinkConfigConstants.PASSWORD_CONFIG)
    .define(VoltSinkConfigConstants.KCQL_CONFIG, Type.STRING, Importance.HIGH, VoltSinkConfigConstants.KCQL_CONFIG_DOC,
      "Connection", 4, ConfigDef.Width.MEDIUM, VoltSinkConfigConstants.KCQL_CONFIG)
    .define(VoltSinkConfigConstants.ERROR_POLICY_CONFIG, Type.STRING, VoltSinkConfigConstants.ERROR_POLICY_DEFAULT, Importance.HIGH, VoltSinkConfigConstants.ERROR_POLICY_DOC,
      "Connection", 5, ConfigDef.Width.MEDIUM, VoltSinkConfigConstants.ERROR_POLICY_CONFIG)
    .define(VoltSinkConfigConstants.ERROR_RETRY_INTERVAL_CONFIG, Type.INT, VoltSinkConfigConstants.ERROR_RETRY_INTERVAL_DEFAULT, Importance.MEDIUM,
      VoltSinkConfigConstants.ERROR_RETRY_INTERVAL_DOC, "Connection", 1, ConfigDef.Width.MEDIUM, VoltSinkConfigConstants.ERROR_RETRY_INTERVAL_CONFIG)
    .define(VoltSinkConfigConstants.NBR_OF_RETRIES_CONFIG, Type.INT, VoltSinkConfigConstants.NBR_OF_RETIRES_DEFAULT, Importance.MEDIUM, VoltSinkConfigConstants.NBR_OF_RETRIES_DOC,
      "Connection", 6, ConfigDef.Width.MEDIUM, VoltSinkConfigConstants.NBR_OF_RETRIES_CONFIG)
    .define(VoltSinkConfigConstants.PROGRESS_COUNTER_ENABLED, Type.BOOLEAN, VoltSinkConfigConstants.PROGRESS_COUNTER_ENABLED_DEFAULT,
        Importance.MEDIUM, VoltSinkConfigConstants.PROGRESS_COUNTER_ENABLED_DOC,
        "Metrics", 1, ConfigDef.Width.MEDIUM, VoltSinkConfigConstants.PROGRESS_COUNTER_ENABLED_DISPLAY)
}


case class VoltSinkConfig(props: util.Map[String, String])
  extends BaseConfig(VoltSinkConfigConstants.CONNECTOR_PREFIX, VoltSinkConfig.config, props)
    with KcqlSettings
    with UserSettings
    with ErrorPolicySettings
    with NumberRetriesSettings

Source File: CoapSinkConnector.scala From stream-reactor with Apache License 2.0

5 votes

package com.datamountaineer.streamreactor.connect.coap.sink

import java.util

import com.datamountaineer.streamreactor.connect.coap.configs.{CoapConstants, CoapSinkConfig}
import com.datamountaineer.streamreactor.connect.config.Helpers
import com.datamountaineer.streamreactor.connect.utils.JarManifest
import com.typesafe.scalalogging.StrictLogging
import org.apache.kafka.common.config.ConfigDef
import org.apache.kafka.connect.connector.Task
import org.apache.kafka.connect.sink.SinkConnector

import scala.collection.JavaConverters._


class CoapSinkConnector extends SinkConnector with StrictLogging {
  private var configProps: util.Map[String, String] = _
  private val configDef = CoapSinkConfig.config
  private val manifest = JarManifest(getClass.getProtectionDomain.getCodeSource.getLocation)

  override def taskClass(): Class[_ <: Task] = classOf[CoapSinkTask]
  override def start(props: util.Map[String, String]): Unit = {
    Helpers.checkInputTopics(CoapConstants.COAP_KCQL, props.asScala.toMap)
    configProps = props
  }

  override def taskConfigs(maxTasks: Int): util.List[util.Map[String, String]] = {
    logger.info(s"Setting task configurations for $maxTasks workers.")
    (1 to maxTasks).map(_ => configProps).toList.asJava
  }

  override def stop(): Unit = {}
  override def config(): ConfigDef = configDef
  override def version(): String = manifest.version()
}

Source File: CoapSourceConnector.scala From stream-reactor with Apache License 2.0

5 votes

package com.datamountaineer.streamreactor.connect.coap.source

import java.util

import com.datamountaineer.streamreactor.connect.coap.configs.{CoapConstants, CoapSourceConfig}
import com.datamountaineer.streamreactor.connect.utils.JarManifest
import org.apache.kafka.common.config.ConfigDef
import org.apache.kafka.connect.connector.Task
import org.apache.kafka.connect.source.SourceConnector
import org.apache.kafka.connect.util.ConnectorUtils

import scala.collection.JavaConverters._


class CoapSourceConnector extends SourceConnector {
  private var configProps: util.Map[String, String] = _
  private val configDef = CoapSourceConfig.config
  private val manifest = JarManifest(getClass.getProtectionDomain.getCodeSource.getLocation)

  override def taskClass(): Class[_ <: Task] = classOf[CoapSourceTask]

  override def taskConfigs(maxTasks: Int): util.List[util.Map[String, String]] = {
    val raw = configProps.get(CoapConstants.COAP_KCQL)
    require(raw != null && !raw.isEmpty,  s"No ${CoapConstants.COAP_KCQL} provided!")

    //sql1, sql2
    val kcqls = raw.split(";")
    val groups = ConnectorUtils.groupPartitions(kcqls.toList.asJava, maxTasks).asScala

    //split up the kcql statement based on the number of tasks.
    groups
      .filterNot(g => g.asScala.isEmpty)
      .map(g => {
        val taskConfigs = new java.util.HashMap[String,String]
        taskConfigs.putAll(configProps)
        taskConfigs.put(CoapConstants.COAP_KCQL, g.asScala.mkString(";")) //overwrite
        taskConfigs.asScala.asJava
      })
  }.asJava

  override def config(): ConfigDef = configDef

  override def start(props: util.Map[String, String]): Unit = configProps = props

  override def stop(): Unit = {}

  override def version(): String = manifest.version()
}

Source File: FtpSourceConfig.scala From stream-reactor with Apache License 2.0

5 votes

package com.datamountaineer.streamreactor.connect.ftp.source

import java.util

import com.datamountaineer.streamreactor.connect.ftp.source.MonitorMode.MonitorMode
import org.apache.kafka.common.config.ConfigDef.{Importance, Type}
import org.apache.kafka.common.config.{AbstractConfig, ConfigDef}

import scala.collection.JavaConverters._

object MonitorMode extends Enumeration {
  type MonitorMode = Value
  val Tail, Update = Value
}

case class MonitorConfig(topic:String, path:String, mode: MonitorMode)

object KeyStyle extends Enumeration {
  type KeyStyle = Value
  val String = Value(FtpSourceConfig.StringKeyStyle)
  val Struct = Value(FtpSourceConfig.StructKeyStyle)
}

object FtpProtocol extends Enumeration {
  type FtpProtocol = Value
  val FTPS, FTP = Value
}

import com.datamountaineer.streamreactor.connect.ftp.source.KeyStyle._

object FtpSourceConfig {
  val Address = "connect.ftp.address"
  val User = "connect.ftp.user"
  val Password = "connect.ftp.password"
  val FtpTimeout = "connect.ftp.timeout"
  val MaxBackoff = "connect.ftp.max.backoff"
  val RefreshRate = "connect.ftp.refresh"
  val MonitorTail = "connect.ftp.monitor.tail"
  val MonitorUpdate = "connect.ftp.monitor.update"
  val MonitorSliceSize = "connect.ftp.monitor.slicesize"
  val FileMaxAge = "connect.ftp.file.maxage"
  val KeyStyle = "connect.ftp.keystyle"
  val StringKeyStyle = "string"
  val StructKeyStyle = "struct"
  val FileConverter = "connect.ftp.fileconverter"
  val SourceRecordConverter = "connect.ftp.sourcerecordconverter"
  val FtpMaxPollRecords = "connect.ftp.max.poll.records"
  val protocol = "connect.ftp.protocol"
  val fileFilter = "connect.ftp.filter"

  val definition: ConfigDef = new ConfigDef()
    .define(Address, Type.STRING, Importance.HIGH, "ftp address[:port]")
    .define(User, Type.STRING, Importance.HIGH, "ftp user name to login")
    .define(Password, Type.PASSWORD, Importance.HIGH, "ftp password to login")
    .define(RefreshRate, Type.STRING, Importance.HIGH, "how often the ftp server is polled; ISO8601 duration")
    .define(MaxBackoff, Type.STRING,"PT30M", Importance.HIGH, "on failure, exponentially backoff to at most this ISO8601 duration")
    .define(FileMaxAge, Type.STRING, Importance.HIGH, "ignore files older than this; ISO8601 duration")
    .define(MonitorTail, Type.LIST, "", Importance.HIGH, "comma separated lists of path:destinationtopic; tail of file is tracked")
    .define(MonitorUpdate, Type.LIST, "", Importance.HIGH, "comma separated lists of path:destinationtopic; whole file is tracked")
    .define(MonitorSliceSize, Type.INT, -1, Importance.HIGH, "slice size in bytes")
    .define(KeyStyle, Type.STRING, Importance.HIGH, s"what the output key is set to: `${StringKeyStyle}` => filename; `${StructKeyStyle}` => structure with filename and offset")
    .define(FileConverter, Type.CLASS, "com.datamountaineer.streamreactor.connect.ftp.source.SimpleFileConverter", Importance.HIGH, s"TODO")
    .define(SourceRecordConverter, Type.CLASS, "com.datamountaineer.streamreactor.connect.ftp.source.NopSourceRecordConverter", Importance.HIGH, s"TODO")
    .define(FtpMaxPollRecords, Type.INT, 10000, Importance.LOW, "Max number of records returned per poll")
    .define(protocol, Type.STRING, "ftp", Importance.LOW, "FTPS or FTP protocol")
    .define(fileFilter, Type.STRING, ".*", Importance.LOW, "Regular expression to use when selecting files for processing ignoring file which do not match")
    .define(FtpTimeout, Type.INT, 30000, Importance.LOW, "Ftp connection timeout in milliseconds")

}

// abstracts the properties away a bit
class FtpSourceConfig(props: util.Map[String, String])
  extends AbstractConfig(FtpSourceConfig.definition, props) {

  // don't leak our ugly config!
  def ftpMonitorConfigs(): Seq[MonitorConfig] = {
    lazy val topicPathRegex = "([^:]*):(.*)".r
    getList(FtpSourceConfig.MonitorTail).asScala.map { case topicPathRegex(path, topic) => MonitorConfig(topic, path, mode = MonitorMode.Tail) } ++
      getList(FtpSourceConfig.MonitorUpdate).asScala.map { case topicPathRegex(path, topic) => MonitorConfig(topic, path, mode = MonitorMode.Update) }
  }

  def address(): (String, Option[Int]) = {
    lazy val hostIpRegex = "([^:]*):?([0-9]*)".r
    val hostIpRegex(host, port) = getString(FtpSourceConfig.Address)
    (host, if (port.isEmpty) None else Some(port.toInt))
  }

  def keyStyle(): KeyStyle = KeyStyle.values.find(_.toString.toLowerCase == getString(FtpSourceConfig.KeyStyle)).get

  def sourceRecordConverter(): SourceRecordConverter =
    getConfiguredInstance(FtpSourceConfig.SourceRecordConverter, classOf[SourceRecordConverter])

  def fileConverter = getClass(FtpSourceConfig.FileConverter)

  def timeoutMs() = getInt(FtpSourceConfig.FtpTimeout)

  def maxPollRecords = getInt(FtpSourceConfig.FtpMaxPollRecords)

  def getProtocol = FtpProtocol.withName(getString(FtpSourceConfig.protocol).toUpperCase)
}

Source File: TwitterSinkConfig.scala From kafka-tweet-producer with Apache License 2.0

5 votes

package com.eneco.trading.kafka.connect.twitter

import java.util

import org.apache.kafka.common.config.{AbstractConfig, ConfigDef}
import org.apache.kafka.common.config.ConfigDef.{Importance, Type}
import org.apache.kafka.connect.sink.SinkTask

object TwitterSinkConfig {
  val CONSUMER_KEY_CONFIG = "twitter.consumerkey"
  val CONSUMER_KEY_CONFIG_DOC = "Twitter account consumer key."
  val CONSUMER_SECRET_CONFIG = "twitter.consumersecret"
  val CONSUMER_SECRET_CONFIG_DOC = "Twitter account consumer secret."
  val TOKEN_CONFIG = "twitter.token"
  val TOKEN_CONFIG_DOC = "Twitter account token."
  val SECRET_CONFIG = "twitter.secret"
  val SECRET_CONFIG_DOC = "Twitter account secret."
  val TOPICS = SinkTask.TOPICS_CONFIG
  val TOPICS_DOC = "The Kafka topic to read from."

  val config: ConfigDef = new ConfigDef()
    .define(CONSUMER_KEY_CONFIG, Type.STRING, Importance.HIGH, CONSUMER_KEY_CONFIG_DOC)
    .define(CONSUMER_SECRET_CONFIG, Type.PASSWORD, Importance.HIGH, CONSUMER_SECRET_CONFIG_DOC)
    .define(TOKEN_CONFIG, Type.STRING, Importance.HIGH, TOKEN_CONFIG_DOC)
    .define(SECRET_CONFIG, Type.PASSWORD, Importance.HIGH, SECRET_CONFIG_DOC)
    .define(TOPICS, Type.LIST, Importance.HIGH, TOPICS_DOC)
}

class TwitterSinkConfig(props: util.Map[String, String])
  extends AbstractConfig(TwitterSinkConfig.config, props) {
}

Source File: TwitterSinkConfig.scala From kafka-connect-twitter with Apache License 2.0

5 votes

package com.eneco.trading.kafka.connect.twitter

import java.util

import org.apache.kafka.common.config.{AbstractConfig, ConfigDef}
import org.apache.kafka.common.config.ConfigDef.{Importance, Type}
import org.apache.kafka.connect.sink.SinkTask

object TwitterSinkConfig {
  val CONSUMER_KEY_CONFIG = "twitter.consumerkey"
  val CONSUMER_KEY_CONFIG_DOC = "Twitter account consumer key."
  val CONSUMER_SECRET_CONFIG = "twitter.consumersecret"
  val CONSUMER_SECRET_CONFIG_DOC = "Twitter account consumer secret."
  val TOKEN_CONFIG = "twitter.token"
  val TOKEN_CONFIG_DOC = "Twitter account token."
  val SECRET_CONFIG = "twitter.secret"
  val SECRET_CONFIG_DOC = "Twitter account secret."
  val TOPICS = SinkTask.TOPICS_CONFIG
  val TOPICS_DOC = "The Kafka topic to read from."

  val config: ConfigDef = new ConfigDef()
    .define(CONSUMER_KEY_CONFIG, Type.STRING, Importance.HIGH, CONSUMER_KEY_CONFIG_DOC)
    .define(CONSUMER_SECRET_CONFIG, Type.PASSWORD, Importance.HIGH, CONSUMER_SECRET_CONFIG_DOC)
    .define(TOKEN_CONFIG, Type.STRING, Importance.HIGH, TOKEN_CONFIG_DOC)
    .define(SECRET_CONFIG, Type.PASSWORD, Importance.HIGH, SECRET_CONFIG_DOC)
    .define(TOPICS, Type.LIST, Importance.HIGH, TOPICS_DOC)
}

class TwitterSinkConfig(props: util.Map[String, String])
  extends AbstractConfig(TwitterSinkConfig.config, props) {
}

Source File: JMSConfig.scala From stream-reactor with Apache License 2.0

5 votes

package com.datamountaineer.streamreactor.connect.jms.config

import java.util

import com.datamountaineer.streamreactor.connect.config.base.traits._
import org.apache.kafka.common.config.ConfigDef
import org.apache.kafka.common.config.ConfigDef.{Importance, Type}

object JMSConfig {

  val config: ConfigDef = new ConfigDef()
    .define(JMSConfigConstants.JMS_URL, Type.STRING, Importance.HIGH, JMSConfigConstants.JMS_URL_DOC,
      "Connection", 1, ConfigDef.Width.MEDIUM, JMSConfigConstants.JMS_URL)
    .define(JMSConfigConstants.INITIAL_CONTEXT_FACTORY, Type.STRING, Importance.HIGH, JMSConfigConstants.INITIAL_CONTEXT_FACTORY_DOC,
      "Connection", 2, ConfigDef.Width.MEDIUM, JMSConfigConstants.INITIAL_CONTEXT_FACTORY)
    .define(JMSConfigConstants.CONNECTION_FACTORY, Type.STRING, JMSConfigConstants.CONNECTION_FACTORY_DEFAULT, Importance.HIGH, JMSConfigConstants.CONNECTION_FACTORY_DOC,
      "Connection", 3, ConfigDef.Width.MEDIUM, JMSConfigConstants.CONNECTION_FACTORY)
    .define(JMSConfigConstants.KCQL, Type.STRING, Importance.HIGH, JMSConfigConstants.KCQL,
      "Connection", 4, ConfigDef.Width.MEDIUM, JMSConfigConstants.KCQL)
    .define(JMSConfigConstants.TOPIC_SUBSCRIPTION_NAME, Type.STRING, null, Importance.HIGH, JMSConfigConstants.TOPIC_SUBSCRIPTION_NAME_DOC,
      "Connection", 5, ConfigDef.Width.MEDIUM, JMSConfigConstants.TOPIC_SUBSCRIPTION_NAME)
    .define(JMSConfigConstants.JMS_PASSWORD, Type.PASSWORD, null, Importance.HIGH, JMSConfigConstants.JMS_PASSWORD_DOC,
      "Connection", 6, ConfigDef.Width.MEDIUM, JMSConfigConstants.JMS_PASSWORD)
    .define(JMSConfigConstants.JMS_USER, Type.STRING, null, Importance.HIGH, JMSConfigConstants.JMS_USER_DOC,
      "Connection", 7, ConfigDef.Width.MEDIUM, JMSConfigConstants.JMS_USER)
    .define(JMSConfigConstants.ERROR_POLICY, Type.STRING, JMSConfigConstants.ERROR_POLICY_DEFAULT, Importance.HIGH,
      JMSConfigConstants.ERROR_POLICY_DOC,
      "Connection", 8, ConfigDef.Width.MEDIUM, JMSConfigConstants.ERROR_POLICY)
    .define(JMSConfigConstants.ERROR_RETRY_INTERVAL, Type.INT, JMSConfigConstants.ERROR_RETRY_INTERVAL_DEFAULT,
      Importance.MEDIUM, JMSConfigConstants.ERROR_RETRY_INTERVAL_DOC,
      "Connection", 9, ConfigDef.Width.MEDIUM, JMSConfigConstants.ERROR_RETRY_INTERVAL)
    .define(JMSConfigConstants.NBR_OF_RETRIES, Type.INT, JMSConfigConstants.NBR_OF_RETIRES_DEFAULT,
      Importance.MEDIUM, JMSConfigConstants.NBR_OF_RETRIES_DOC,
      "Connection", 10, ConfigDef.Width.MEDIUM, JMSConfigConstants.NBR_OF_RETRIES)
    .define(JMSConfigConstants.DESTINATION_SELECTOR, Type.STRING, JMSConfigConstants.DESTINATION_SELECTOR_DEFAULT,
      Importance.MEDIUM, JMSConfigConstants.DESTINATION_SELECTOR_DOC,
      "Connection", 11, ConfigDef.Width.MEDIUM, JMSConfigConstants.DESTINATION_SELECTOR)
    .define(JMSConfigConstants.EXTRA_PROPS, Type.LIST, JMSConfigConstants.EXTRA_PROPS_DEFAULT, Importance.MEDIUM,
      JMSConfigConstants.EXTRA_PROPS_DOC,
      "Connection", 12, ConfigDef.Width.MEDIUM, JMSConfigConstants.EXTRA_PROPS)
    .define(JMSConfigConstants.BATCH_SIZE, Type.INT, JMSConfigConstants.BATCH_SIZE_DEFAULT, Importance.MEDIUM,
      JMSConfigConstants.BATCH_SIZE_DOC,
      "Connection", 13, ConfigDef.Width.MEDIUM, JMSConfigConstants.BATCH_SIZE)
    .define(JMSConfigConstants.POLLING_TIMEOUT_CONFIG, Type.LONG, JMSConfigConstants.POLLING_TIMEOUT_DEFAULT, Importance.MEDIUM,
      JMSConfigConstants.POLLING_TIMEOUT_DOC,
      "Connection", 14, ConfigDef.Width.MEDIUM, JMSConfigConstants.POLLING_TIMEOUT_CONFIG)

    //converters

    .define(JMSConfigConstants.DEFAULT_CONVERTER_CONFIG, Type.STRING, "", Importance.HIGH, JMSConfigConstants.DEFAULT_CONVERTER_DOC,
      "Converter", 1, ConfigDef.Width.MEDIUM, JMSConfigConstants.DEFAULT_CONVERTER_DISPLAY)
    .define(JMSConfigConstants.THROW_ON_CONVERT_ERRORS_CONFIG, Type.BOOLEAN, JMSConfigConstants.THROW_ON_CONVERT_ERRORS_DEFAULT,
      Importance.HIGH, JMSConfigConstants.THROW_ON_CONVERT_ERRORS_DOC, "Converter", 2, ConfigDef.Width.MEDIUM,
      JMSConfigConstants.THROW_ON_CONVERT_ERRORS_DISPLAY)
    .define(JMSConfigConstants.AVRO_CONVERTERS_SCHEMA_FILES, Type.STRING, JMSConfigConstants.AVRO_CONVERTERS_SCHEMA_FILES_DEFAULT,
      Importance.HIGH, JMSConfigConstants.AVRO_CONVERTERS_SCHEMA_FILES_DOC, "Converter", 3, ConfigDef.Width.MEDIUM,
      JMSConfigConstants.AVRO_CONVERTERS_SCHEMA_FILES)
    .define(JMSConfigConstants.HEADERS_CONFIG,
      Type.STRING, "", Importance.LOW, JMSConfigConstants.HEADERS_CONFIG_DOC,
      "Converter", 4, ConfigDef.Width.MEDIUM, JMSConfigConstants.HEADERS_CONFIG_DISPLAY)

    .define(JMSConfigConstants.PROGRESS_COUNTER_ENABLED, Type.BOOLEAN, JMSConfigConstants.PROGRESS_COUNTER_ENABLED_DEFAULT,
      Importance.MEDIUM, JMSConfigConstants.PROGRESS_COUNTER_ENABLED_DOC,
      "Metrics", 1, ConfigDef.Width.MEDIUM, JMSConfigConstants.PROGRESS_COUNTER_ENABLED_DISPLAY)

    .define(JMSConfigConstants.EVICT_UNCOMMITTED_MINUTES,
      Type.INT,
      JMSConfigConstants.EVICT_UNCOMMITTED_MINUTES_DEFAULT,
      Importance.MEDIUM, JMSConfigConstants.EVICT_UNCOMMITTED_MINUTES_DOC,
      "Settings", 1, ConfigDef.Width.MEDIUM, JMSConfigConstants.EVICT_UNCOMMITTED_MINUTES_DOC)

    .define(JMSConfigConstants.EVICT_THRESHOLD_MINUTES,
      Type.INT,
      JMSConfigConstants.EVICT_THRESHOLD_MINUTES_DEFAULT,
      Importance.MEDIUM, JMSConfigConstants.EVICT_THRESHOLD_MINUTES_DOC,
      "Settings", 2, ConfigDef.Width.MEDIUM, JMSConfigConstants.EVICT_THRESHOLD_MINUTES_DOC)

    .define(JMSConfigConstants.TASK_PARALLELIZATION_TYPE,
        Type.STRING,
        JMSConfigConstants.TASK_PARALLELIZATION_TYPE_DEFAULT,
        Importance.MEDIUM, JMSConfigConstants.TASK_PARALLELIZATION_TYPE_DOC,
        "Settings", 4, ConfigDef.Width.MEDIUM, JMSConfigConstants.TASK_PARALLELIZATION_TYPE_DOC)
}


case class JMSConfig(props: util.Map[String, String])
  extends BaseConfig(JMSConfigConstants.CONNECTOR_PREFIX, JMSConfig.config, props)
    with KcqlSettings
    with ErrorPolicySettings
    with NumberRetriesSettings
    with UserSettings
    with ConnectionSettings

Source File: IotHubSinkConfig.scala From toketi-kafka-connect-iothub with MIT License

5 votes

package com.microsoft.azure.iot.kafka.connect.sink

import java.util.Map

import com.microsoft.azure.sdk.iot.service.DeliveryAcknowledgement
import org.apache.kafka.common.config.ConfigDef.{Importance, Type, Width}
import org.apache.kafka.common.config.{AbstractConfig, ConfigDef}

object IotHubSinkConfig {

  val IotHubConnectionString    = "IotHub.ConnectionString"
  val IotHubMessageDeliveryAcknowledgement = "IotHub.MessageDeliveryAcknowledgement"

  private val IotHubConnectionStringDoc =
    """IoT Hub ConnectionString. (see "IoT Hub" >> your hub >> "Shared access policies" >> "service" >> """ +
      """"Connection string")"""
  private val IotHubMessageDeliveryAcknowledgementDoc = "The type of delivery acknowledgement for a C2D message. " +
    "Valid values are None, Full, NegativeOnly, PositiveOnly"
  private val iotConfigGroup = "Azure IoT Hub"
  private val validDeliveryAcknowledgementString = ConfigDef.ValidString.in(
    DeliveryAcknowledgement.None.toString,
    DeliveryAcknowledgement.Full.toString,
    DeliveryAcknowledgement.PositiveOnly.toString,
    DeliveryAcknowledgement.NegativeOnly.toString)

  lazy val configDef = new ConfigDef()
    .define(IotHubConnectionString, Type.STRING, Importance.HIGH, IotHubConnectionStringDoc, iotConfigGroup, 1,
      Width.MEDIUM, "IoT Hub Connection String")
    .define(IotHubMessageDeliveryAcknowledgement, Type.STRING, DeliveryAcknowledgement.None.toString,
      validDeliveryAcknowledgementString, Importance.HIGH, IotHubMessageDeliveryAcknowledgementDoc, iotConfigGroup, 1,
      Width.MEDIUM, "Delivery acknowledgement")

  def getConfig(configValues: Map[String, String]): IotHubSinkConfig = {
    new IotHubSinkConfig(configDef, configValues)
  }
}

class IotHubSinkConfig(configDef: ConfigDef, configValues: Map[String, String])
  extends AbstractConfig(configDef, configValues)

Source File: IotHubSinkConnector.scala From toketi-kafka-connect-iothub with MIT License

5 votes

package com.microsoft.azure.iot.kafka.connect.sink

import java.util

import com.microsoft.azure.iot.kafka.connect.source.JsonSerialization
import com.typesafe.scalalogging.LazyLogging
import org.apache.kafka.common.config.{ConfigDef, ConfigException}
import org.apache.kafka.connect.connector.Task
import org.apache.kafka.connect.errors.ConnectException
import org.apache.kafka.connect.sink.SinkConnector

import scala.collection.JavaConverters._

class IotHubSinkConnector extends SinkConnector with LazyLogging with JsonSerialization {

  private[this] var props: Map[String, String] = _

  override def taskClass(): Class[_ <: Task] = classOf[IotHubSinkTask]

  override def taskConfigs(maxTasks: Int): util.List[util.Map[String, String]] = {
    (1 to maxTasks).map(_ => this.props.asJava).toList.asJava
  }

  override def stop(): Unit = {
    logger.info("Stopping IotHubSinkConnector")
  }

  override def config(): ConfigDef = IotHubSinkConfig.configDef

  override def start(props: util.Map[String, String]): Unit = {

    logger.info("Starting IotHubSinkConnector")

    try {
      val iotHubSinkConfig = IotHubSinkConfig.getConfig(props)
      this.props = Map[String, String](
        IotHubSinkConfig.IotHubConnectionString -> iotHubSinkConfig.getString(IotHubSinkConfig.IotHubConnectionString),
        IotHubSinkConfig.IotHubMessageDeliveryAcknowledgement →
          iotHubSinkConfig.getString(IotHubSinkConfig.IotHubMessageDeliveryAcknowledgement)
      )
    } catch {
      case ex: ConfigException ⇒ throw new ConnectException("Could not start IotHubSinkConnector due to a " +
        "configuration exception", ex)
    }
  }

  override def version(): String = getClass.getPackage.getImplementationVersion
}

Source File: IotHubSourceConfig.scala From toketi-kafka-connect-iothub with MIT License

5 votes

// Copyright (c) Microsoft. All rights reserved.

package com.microsoft.azure.iot.kafka.connect.source

import com.microsoft.azure.eventhubs.EventHubClient
import org.apache.kafka.common.config.ConfigDef.{Importance, Type, Width}
import org.apache.kafka.common.config.{AbstractConfig, ConfigDef}

import java.util.Map

object IotHubSourceConfig {

  private val defaultBatchSize = 100
  private val defaultReceiveTimeout = 60
  private val iotConfigGroup   = "Azure IoT Hub"
  private val kafkaConfig      = "Kafka"

  val EventHubCompatibleConnectionString = "IotHub.EventHubCompatibleConnectionString"
  val EventHubCompatibleName             = "IotHub.EventHubCompatibleName"
  val EventHubCompatibleNameDoc          =
    """EventHub compatible name ("IoT Hub" >> your hub >> "Endpoints" >> "Events" >> "Event Hub-compatible name")"""
  val EventHubCompatibleEndpoint        = "IotHub.EventHubCompatibleEndpoint"
  val EventHubCompatibleEndpointDoc     =
    """EventHub compatible endpoint ("IoT Hub" >> your hub >> "Endpoints" >> "Events" >> "Event Hub-compatible """ +
      """endpoint")"""
  val IotHubAccessKeyName                = "IotHub.AccessKeyName"
  val IotHubAccessKeyNameDoc             =
    """IotHub access key name ("IoT Hub" >> your hub >> "Shared access policies", default is service)"""
  val IotHubAccessKeyValue               = "IotHub.AccessKeyValue"
  val IotHubAccessKeyValueDoc            =
    """IotHub access key value ("IoT Hub" >> your hub >> "Shared access policies" >> key name >> "Primary key")"""
  val IotHubConsumerGroup                = "IotHub.ConsumerGroup"
  val IotHubConsumerGroupDoc             = "The IoT Hub consumer group"
  val IotHubPartitions                   = "IotHub.Partitions"
  val IotHubPartitionsDoc                = "Number of IoT Hub partitions"
  val KafkaTopic                         = "Kafka.Topic"
  val KafkaTopicDoc                      = "Kafka topic to copy data to"
  val BatchSize                          = "BatchSize"
  val BatchSizeDoc                       = "The batch size for fetching records from IoT Hub"
  val ReceiveTimeout                     = "ReceiveTimeout"
  val ReceiveTimeoutDoc                  = "Max time to spend receiving messages from IoT Hub"
  val IotHubOffset                       = "IotHub.Offsets"
  val IotHubOffsetDoc                    =
    "Offset for each partition in IotHub, as a comma separated string. This value is ignored if IotHubStartTime is specified."
  val IotHubStartTime                    = "IotHub.StartTime"
  val IotHubStartTimeDoc                 = "The time after which to process messages from IoT Hub If this value " +
    "is specified, IotHubOffset value is ignored."
  val TaskPartitionOffsetsMap            = "TaskPartitions"

  lazy val configDef = new ConfigDef()
    .define(EventHubCompatibleName, Type.STRING, Importance.HIGH, EventHubCompatibleNameDoc, iotConfigGroup, 1, Width
      .MEDIUM, "Event Hub compatible name")
    .define(EventHubCompatibleEndpoint, Type.STRING, Importance.HIGH, EventHubCompatibleEndpointDoc,
      iotConfigGroup, 2, Width.MEDIUM, "Event Hub compatible endpoint")
    .define(IotHubAccessKeyName, Type.STRING, Importance.HIGH, IotHubAccessKeyNameDoc, iotConfigGroup, 3, Width.SHORT,
      "Access key name")
    .define(IotHubAccessKeyValue, Type.STRING, Importance.HIGH, IotHubAccessKeyValueDoc, iotConfigGroup, 4,
      Width.LONG, "Access key value")
    .define(IotHubConsumerGroup, Type.STRING, EventHubClient.DEFAULT_CONSUMER_GROUP_NAME, Importance.MEDIUM,
      IotHubConsumerGroupDoc, iotConfigGroup, 5, Width.SHORT, "Consumer group")
    .define(IotHubPartitions, Type.INT, Importance.HIGH, IotHubPartitionsDoc, iotConfigGroup, 6, Width.SHORT,
      "IoT Hub partitions")
    .define(IotHubStartTime, Type.STRING, "", Importance.MEDIUM, IotHubStartTimeDoc, iotConfigGroup, 7, Width.MEDIUM,
      "Start time")
    .define(IotHubOffset, Type.STRING, "", Importance.MEDIUM, IotHubOffsetDoc, iotConfigGroup, 8, Width.MEDIUM,
      "Per partition offsets")
    .define(BatchSize, Type.INT, defaultBatchSize, Importance.MEDIUM, IotHubOffsetDoc, iotConfigGroup, 9, Width.SHORT,
      "Batch size")
    .define(ReceiveTimeout, Type.INT, defaultReceiveTimeout, Importance.MEDIUM, ReceiveTimeoutDoc, iotConfigGroup, 10,
      Width.SHORT, "Receive Timeout")
    .define(KafkaTopic, Type.STRING, Importance.HIGH, KafkaTopicDoc, kafkaConfig, 11, Width.MEDIUM, "Kafka topic")

  def getConfig(configValues: Map[String, String]): IotHubSourceConfig = {
    new IotHubSourceConfig(configDef, configValues)
  }
}

class IotHubSourceConfig(configDef: ConfigDef, configValues: Map[String, String])
  extends AbstractConfig(configDef, configValues)

Source File: Conf.scala From sqs-kafka-connect with Apache License 2.0

5 votes

package com.hivehome.kafka.connect.sqs

import org.apache.kafka.common.config.ConfigDef
import org.apache.kafka.common.config.ConfigDef.{Importance, Type}
import org.apache.kafka.connect.errors.ConnectException

import scala.util.Try

case class Conf(queueName: Option[String] = None,
                topicName: Option[String] = None,
                awsRegion: String = "eu-west-1",
                awsKey: Option[String] = None,
                awsSecret: Option[String] = None) {
  def toMap: Map[String, String] = {
    import Conf._
    Map[String, Option[String]](
      SourceSqsQueue -> queueName,
      DestinationKafkaTopic -> topicName,
      AwsKey -> awsKey,
      AwsSecret -> awsSecret,
      AwsRegion -> Some(awsRegion)
    ).collect { case (k, Some(v)) => (k, v) }
  }
}

object Conf {
  val DestinationKafkaTopic = "destination.topic"
  val SourceSqsQueue = "source.queue"
  val AwsKey = "aws.key"
  val AwsSecret = "aws.secret"
  val AwsRegion = "aws.region"

  val ConfigDef = new ConfigDef()
    .define(SourceSqsQueue, Type.STRING, Importance.HIGH, "Source SQS queue name to consumer from.")
    .define(DestinationKafkaTopic, Type.STRING, Importance.HIGH, "Destination Kafka topicName to publish data to")
    .define(AwsKey, Type.STRING, "", Importance.MEDIUM, "AWS Key to connect to SQS")
    .define(AwsSecret, Type.STRING, "", Importance.MEDIUM, "AWS secret to connect to SQS")

  def parse(props: Map[String, String]): Try[Conf] = Try {
    val queueName = props.get(Conf.SourceSqsQueue)
    val topicName = props.get(Conf.DestinationKafkaTopic)
    val awsKey = props.get(Conf.AwsKey).filter(_.nonEmpty)
    val awsSecret = props.get(Conf.AwsSecret).filter(_.nonEmpty)
    val awsRegion = props.get(Conf.AwsRegion)

    if (queueName == null || queueName.isEmpty)
      throw new ConnectException("Configuration must include 'queueName' setting")
    if (queueName.contains(","))
      throw new ConnectException("Configuration should only have a single queueName when used as a source.")
    if (topicName == null || topicName.isEmpty)
      throw new ConnectException("Configuration must include 'topicName' setting")

    val conf = Conf(queueName = queueName, topicName = topicName, awsKey = awsKey, awsSecret = awsSecret)
    awsRegion match {
      case Some(region) => conf.copy(awsRegion = region)
      case _ => conf
    }
  }
}

Source File: DocumentDbConfig.scala From stream-reactor with Apache License 2.0

5 votes

package com.datamountaineer.streamreactor.connect.azure.documentdb.config

import java.util

import com.datamountaineer.streamreactor.connect.config.base.traits._
import com.microsoft.azure.documentdb.ConsistencyLevel
import org.apache.kafka.common.config.ConfigDef
import org.apache.kafka.common.config.ConfigDef.{Importance, Type}

object DocumentDbConfig {
  val config: ConfigDef = new ConfigDef()
    .define(DocumentDbConfigConstants.CONNECTION_CONFIG, Type.STRING, Importance.HIGH,
      DocumentDbConfigConstants.CONNECTION_CONFIG_DOC, "Connection", 1, ConfigDef.Width.LONG,
      DocumentDbConfigConstants.CONNECTION_DISPLAY)
    .define(DocumentDbConfigConstants.MASTER_KEY_CONFIG, Type.PASSWORD, Importance.HIGH,
      DocumentDbConfigConstants.MASTER_KEY_DOC, "Connection", 2, ConfigDef.Width.LONG,
      DocumentDbConfigConstants.MASTER_KEY_CONFIG)
    .define(DocumentDbConfigConstants.CONSISTENCY_CONFIG, Type.STRING, DocumentDbConfigConstants.CONSISTENCY_DEFAULT,
      Importance.HIGH, DocumentDbConfigConstants.CONSISTENCY_DOC, "Connection", 3, ConfigDef.Width.LONG,
      DocumentDbConfigConstants.CONSISTENCY_DISPLAY)
    .define(DocumentDbConfigConstants.DATABASE_CONFIG, Type.STRING, Importance.HIGH,
      DocumentDbConfigConstants.DATABASE_CONFIG_DOC, "Connection", 4, ConfigDef.Width.MEDIUM,
      DocumentDbConfigConstants.DATABASE_CONFIG)
    .define(DocumentDbConfigConstants.CREATE_DATABASE_CONFIG, Type.BOOLEAN,
      DocumentDbConfigConstants.CREATE_DATABASE_DEFAULT, Importance.MEDIUM,
      DocumentDbConfigConstants.CREATE_DATABASE_DOC, "Connection", 5, ConfigDef.Width.MEDIUM,
      DocumentDbConfigConstants.CREATE_DATABASE_DISPLAY)
    .define(DocumentDbConfigConstants.PROXY_HOST_CONFIG, Type.STRING, null, Importance.LOW,
      DocumentDbConfigConstants.PROXY_HOST_DOC, "Connection", 5, ConfigDef.Width.MEDIUM,
      DocumentDbConfigConstants.PROXY_HOST_DISPLAY)
    .define(DocumentDbConfigConstants.KCQL_CONFIG, Type.STRING, Importance.HIGH, DocumentDbConfigConstants.KCQL_DOC,
      "Mappings", 1, ConfigDef.Width.LONG, DocumentDbConfigConstants.KCQL_CONFIG)
    .define(DocumentDbConfigConstants.ERROR_POLICY_CONFIG, Type.STRING, DocumentDbConfigConstants.ERROR_POLICY_DEFAULT,
      Importance.HIGH, DocumentDbConfigConstants.ERROR_POLICY_DOC, "Error", 1, ConfigDef.Width.LONG,
      DocumentDbConfigConstants.ERROR_POLICY_CONFIG)
    .define(DocumentDbConfigConstants.NBR_OF_RETRIES_CONFIG, Type.INT, DocumentDbConfigConstants.NBR_OF_RETIRES_DEFAULT,
      Importance.MEDIUM, DocumentDbConfigConstants.NBR_OF_RETRIES_DOC, "Error", 2, ConfigDef.Width.LONG,
      DocumentDbConfigConstants.NBR_OF_RETRIES_CONFIG)
    .define(DocumentDbConfigConstants.ERROR_RETRY_INTERVAL_CONFIG, Type.INT,
      DocumentDbConfigConstants.ERROR_RETRY_INTERVAL_DEFAULT, Importance.MEDIUM,
      DocumentDbConfigConstants.ERROR_RETRY_INTERVAL_DOC, "Error", 3, ConfigDef.Width.LONG,
      DocumentDbConfigConstants.ERROR_RETRY_INTERVAL_CONFIG)
    .define(DocumentDbConfigConstants.PROGRESS_COUNTER_ENABLED, Type.BOOLEAN,
      DocumentDbConfigConstants.PROGRESS_COUNTER_ENABLED_DEFAULT, Importance.MEDIUM,
      DocumentDbConfigConstants.PROGRESS_COUNTER_ENABLED_DOC, "Metrics", 1, ConfigDef.Width.MEDIUM,
      DocumentDbConfigConstants.PROGRESS_COUNTER_ENABLED_DISPLAY)
}

case class DocumentDbConfig(props: util.Map[String, String])
  extends BaseConfig(DocumentDbConfigConstants.CONNECTOR_PREFIX, DocumentDbConfig.config, props)
    with KcqlSettings
    with DatabaseSettings
    with NumberRetriesSettings
    with ErrorPolicySettings
    with ConsistencyLevelSettings[ConsistencyLevel]

Source File: MongoSinkConnector.scala From stream-reactor with Apache License 2.0

5 votes

package com.datamountaineer.streamreactor.connect.mongodb.sink

import java.util

import com.datamountaineer.streamreactor.connect.config.Helpers
import com.datamountaineer.streamreactor.connect.mongodb.config.{MongoConfig, MongoConfigConstants}
import com.datamountaineer.streamreactor.connect.utils.JarManifest
import com.typesafe.scalalogging.StrictLogging
import org.apache.kafka.common.config.{Config, ConfigDef}
import org.apache.kafka.connect.connector.Task
import org.apache.kafka.connect.errors.ConnectException
import org.apache.kafka.connect.sink.SinkConnector

import scala.collection.JavaConverters._
import scala.util.{Failure, Try}


  override def start(props: util.Map[String, String]): Unit = {
    Helpers.checkInputTopics(MongoConfigConstants.KCQL_CONFIG, props.asScala.toMap)
    Try(MongoConfig(props)) match {
      case Failure(f) => throw new ConnectException(s"Couldn't start Mongo sink due to configuration error: ${f.getMessage}", f)
      case _ =>
    }

    configProps = props
  }

  override def stop(): Unit = {}

  override def version(): String = manifest.version()

  override def config(): ConfigDef = MongoConfig.config

  override def validate(connectorConfigs: util.Map[String, String]): Config = super.validate(connectorConfigs)
}

Source File: MongoConfig.scala From stream-reactor with Apache License 2.0

5 votes

package com.datamountaineer.streamreactor.connect.mongodb.config

import java.util

import com.datamountaineer.streamreactor.connect.config.base.traits._
import org.apache.kafka.common.config.ConfigDef
import org.apache.kafka.common.config.ConfigDef.{Importance, Type}

object MongoConfig {

  val config = new ConfigDef()
    .define(MongoConfigConstants.CONNECTION_CONFIG, Type.STRING, Importance.HIGH,
      MongoConfigConstants.CONNECTION_CONFIG_DOC, "Connection", 1, ConfigDef.Width.LONG,
      MongoConfigConstants.CONNECTION_CONFIG)
    .define(MongoConfigConstants.DATABASE_CONFIG, Type.STRING, Importance.HIGH,
      MongoConfigConstants.DATABASE_CONFIG_DOC, "Connection", 2, ConfigDef.Width.MEDIUM,
      MongoConfigConstants.DATABASE_CONFIG)


    .define(MongoConfigConstants.USERNAME_CONFIG, Type.STRING, MongoConfigConstants.USERNAME_CONFIG_DEFAULT,
      Importance.MEDIUM, MongoConfigConstants.USERNAME_CONFIG_DOC, "Connection", 3, ConfigDef.Width.MEDIUM,
      MongoConfigConstants.USERNAME_CONFIG)
    .define(MongoConfigConstants.PASSWORD_CONFIG, Type.PASSWORD, MongoConfigConstants.PASSWORD_CONFIG_DEFAULT,
      Importance.MEDIUM, MongoConfigConstants.PASSWORD_CONFIG_DOC, "Connection", 4, ConfigDef.Width.MEDIUM,
      MongoConfigConstants.PASSWORD_CONFIG)
    .define(MongoConfigConstants.AUTHENTICATION_MECHANISM, Type.STRING, MongoConfigConstants.AUTHENTICATION_MECHANISM_DEFAULT,
      Importance.MEDIUM, MongoConfigConstants.AUTHENTICATION_MECHANISM_DOC, "Connection", 5, ConfigDef.Width.MEDIUM,
      MongoConfigConstants.AUTHENTICATION_MECHANISM)

    .define(MongoConfigConstants.KCQL_CONFIG, Type.STRING, Importance.HIGH, MongoConfigConstants.KCQL_DOC,
      "Mappings", 1, ConfigDef.Width.LONG, MongoConfigConstants.KCQL_CONFIG)
    .define(MongoConfigConstants.JSON_DATETIME_FIELDS_CONFIG, Type.LIST, "",
      Importance.LOW, MongoConfigConstants.JSON_DATETIME_FIELDS_DOC, "Mappings", 1, ConfigDef.Width.NONE,
      MongoConfigConstants.JSON_DATETIME_FIELDS_CONFIG)
    .define(MongoConfigConstants.ERROR_POLICY_CONFIG, Type.STRING, MongoConfigConstants.ERROR_POLICY_DEFAULT,
      Importance.HIGH, MongoConfigConstants.ERROR_POLICY_DOC, "Error", 1, ConfigDef.Width.LONG,
      MongoConfigConstants.ERROR_POLICY_CONFIG)
    .define(MongoConfigConstants.NBR_OF_RETRIES_CONFIG, Type.INT, MongoConfigConstants.NBR_OF_RETIRES_DEFAULT,
      Importance.MEDIUM, MongoConfigConstants.NBR_OF_RETRIES_DOC, "Error", 2, ConfigDef.Width.LONG,
      MongoConfigConstants.NBR_OF_RETRIES_CONFIG)
    .define(MongoConfigConstants.ERROR_RETRY_INTERVAL_CONFIG, Type.INT,
      MongoConfigConstants.ERROR_RETRY_INTERVAL_DEFAULT, Importance.MEDIUM,
      MongoConfigConstants.ERROR_RETRY_INTERVAL_DOC, "Error", 3, ConfigDef.Width.LONG,
      MongoConfigConstants.ERROR_RETRY_INTERVAL_CONFIG)
    .define(MongoConfigConstants.PROGRESS_COUNTER_ENABLED, Type.BOOLEAN,
      MongoConfigConstants.PROGRESS_COUNTER_ENABLED_DEFAULT,
      Importance.MEDIUM, MongoConfigConstants.PROGRESS_COUNTER_ENABLED_DOC,
      "Metrics", 1, ConfigDef.Width.MEDIUM, MongoConfigConstants.PROGRESS_COUNTER_ENABLED_DISPLAY)
    .withClientSslSupport()
}


case class MongoConfig(props: util.Map[String, String])
  extends BaseConfig(MongoConfigConstants.CONNECTOR_PREFIX, MongoConfig.config, props)
  with KcqlSettings
  with DatabaseSettings
  with ErrorPolicySettings
  with NumberRetriesSettings
  with UserSettings

Source File: CassandraSinkConnector.scala From stream-reactor with Apache License 2.0

5 votes

package com.datamountaineer.streamreactor.connect.cassandra.sink

import java.util

import com.datamountaineer.streamreactor.connect.cassandra.config.{CassandraConfigConstants, CassandraConfigSink}
import com.datamountaineer.streamreactor.connect.config.Helpers
import com.datamountaineer.streamreactor.connect.utils.JarManifest
import com.typesafe.scalalogging.StrictLogging
import org.apache.kafka.common.config.ConfigDef
import org.apache.kafka.connect.connector.Task
import org.apache.kafka.connect.errors.ConnectException
import org.apache.kafka.connect.sink.SinkConnector

import scala.collection.JavaConverters._
import scala.util.{Failure, Try}


  override def start(props: util.Map[String, String]): Unit = {
    //check input topics
    Helpers.checkInputTopics(CassandraConfigConstants.KCQL, props.asScala.toMap)
    configProps = props
    Try(new CassandraConfigSink(props)) match {
      case Failure(f) =>
        throw new ConnectException(s"Couldn't start Cassandra sink due to configuration error: ${f.getMessage}", f)
      case _ =>
    }
  }

  override def stop(): Unit = {}

  override def version(): String = manifest.version()

  override def config(): ConfigDef = configDef
}

Source File: JMSSourceConnector.scala From stream-reactor with Apache License 2.0

5 votes

package com.datamountaineer.streamreactor.connect.jms.source

import java.util

import com.datamountaineer.streamreactor.connect.jms.config.{JMSConfig, JMSConfigConstants}
import com.datamountaineer.streamreactor.connect.utils.JarManifest
import com.typesafe.scalalogging.StrictLogging
import org.apache.kafka.common.config.ConfigDef
import org.apache.kafka.connect.connector.Task
import org.apache.kafka.connect.source.SourceConnector
import org.apache.kafka.connect.util.ConnectorUtils

import scala.collection.JavaConverters._


class JMSSourceConnector extends SourceConnector with StrictLogging {
  private var configProps: util.Map[String, String] = _
  private val configDef = JMSConfig.config
  private val manifest = JarManifest(getClass.getProtectionDomain.getCodeSource.getLocation)

  override def taskClass(): Class[_ <: Task] = classOf[JMSSourceTask]

  def kcqlTaskScaling(maxTasks: Int): util.List[util.Map[String, String]] = {
    val raw = configProps.get(JMSConfigConstants.KCQL)
    require(raw != null && !raw.isEmpty, s"No ${JMSConfigConstants.KCQL} provided!")

    //sql1, sql2
    val kcqls = raw.split(";")
    val groups = ConnectorUtils.groupPartitions(kcqls.toList.asJava, maxTasks).asScala

    //split up the kcql statement based on the number of tasks.
    groups
      .filterNot(_.isEmpty)
      .map { g =>
        val taskConfigs = new java.util.HashMap[String, String]
        taskConfigs.putAll(configProps)
        taskConfigs.put(JMSConfigConstants.KCQL, g.asScala.mkString(";")) //overwrite
        taskConfigs.asScala.toMap.asJava
      }
  }.asJava

  def defaultTaskScaling(maxTasks: Int): util.List[util.Map[String, String]] = {
    val raw = configProps.get(JMSConfigConstants.KCQL)
    require(raw != null && !raw.isEmpty, s"No ${JMSConfigConstants.KCQL} provided!")
    (1 to maxTasks).map { _ =>
      val taskConfigs: util.Map[String, String] = new java.util.HashMap[String, String]
      taskConfigs.putAll(configProps)
      taskConfigs
    }.toList.asJava
  }

  override def taskConfigs(maxTasks: Int): util.List[util.Map[String, String]] = {
    val config = new JMSConfig(configProps)
    val scaleType = config.getString(JMSConfigConstants.TASK_PARALLELIZATION_TYPE).toLowerCase()
    if (scaleType == JMSConfigConstants.TASK_PARALLELIZATION_TYPE_DEFAULT) {
      kcqlTaskScaling(maxTasks)
    } else defaultTaskScaling(maxTasks)
  }

  override def config(): ConfigDef = configDef

  override def start(props: util.Map[String, String]): Unit = {
    val config = new JMSConfig(props)
    configProps = config.props
  }

  override def stop(): Unit = {}

  override def version(): String = manifest.version()
}

Source File: Validator.scala From ohara with Apache License 2.0

5 votes

package oharastream.ohara.connector.validation

import java.util

import oharastream.ohara.client.configurator.InspectApi
import oharastream.ohara.common.util.VersionUtils
import org.apache.kafka.common.config.ConfigDef
import org.apache.kafka.common.config.ConfigDef.{Importance, Type}
import org.apache.kafka.connect.connector.Task
import org.apache.kafka.connect.source.SourceConnector

import scala.jdk.CollectionConverters._


class Validator extends SourceConnector {
  private[this] var props: util.Map[String, String] = _
  override def version(): String                    = VersionUtils.VERSION
  override def start(props: util.Map[String, String]): Unit = {
    this.props = new util.HashMap[String, String](props)
    // we don't want to make any exception here
  }

  override def taskClass(): Class[_ <: Task] = classOf[ValidatorTask]

  override def taskConfigs(maxTasks: Int): util.List[util.Map[String, String]] =
    Seq.fill(maxTasks)(new util.HashMap[String, String](props)).map(_.asInstanceOf[util.Map[String, String]]).asJava

  override def stop(): Unit = {
    // do nothing
  }
  override def config(): ConfigDef =
    new ConfigDef().define(InspectApi.TARGET_KEY, Type.STRING, null, Importance.HIGH, "target type")
}

Source File: HBaseConfig.scala From stream-reactor with Apache License 2.0

5 votes

package com.datamountaineer.streamreactor.connect.hbase.config

import java.util

import com.datamountaineer.streamreactor.connect.config.base.traits._
import com.datamountaineer.streamreactor.connect.hbase.config.HBaseConfigConstants.CONNECTOR_PREFIX
import org.apache.kafka.common.config.ConfigDef
import org.apache.kafka.common.config.ConfigDef.{Importance, Type}

object HBaseConfig {
    import HBaseConfigConstants._

  val config: ConfigDef = new ConfigDef()
    .define(COLUMN_FAMILY, Type.STRING, Importance.HIGH, COLUMN_FAMILY_DOC,
      "Connection", 1, ConfigDef.Width.MEDIUM, COLUMN_FAMILY)
    .define(KCQL_QUERY, Type.STRING, Importance.HIGH, KCQL_QUERY,
      "Connection", 2, ConfigDef.Width.MEDIUM, KCQL_QUERY)
    .define(ERROR_POLICY, Type.STRING, ERROR_POLICY_DEFAULT, Importance.HIGH, ERROR_POLICY_DOC,
      "Connection", 3, ConfigDef.Width.MEDIUM, ERROR_POLICY)
    .define(ERROR_RETRY_INTERVAL, Type.INT, ERROR_RETRY_INTERVAL_DEFAULT, Importance.MEDIUM, ERROR_RETRY_INTERVAL_DOC,
      "Connection", 4, ConfigDef.Width.MEDIUM, ERROR_RETRY_INTERVAL)
    .define(NBR_OF_RETRIES, Type.INT, NBR_OF_RETIRES_DEFAULT, Importance.MEDIUM, NBR_OF_RETRIES_DOC,
      "Connection", 5, ConfigDef.Width.MEDIUM, NBR_OF_RETRIES)
    .define(PROGRESS_COUNTER_ENABLED, Type.BOOLEAN, PROGRESS_COUNTER_ENABLED_DEFAULT,
      Importance.MEDIUM, PROGRESS_COUNTER_ENABLED_DOC,
      "Metrics", 1, ConfigDef.Width.MEDIUM, PROGRESS_COUNTER_ENABLED_DISPLAY)

    //config folders
    .define(HBASE_CONFIG_DIR, Type.STRING, HBASE_CONFIG_DIR_DEFAULT, Importance.MEDIUM, HBASE_CONFIG_DIR_DOC, "Configs", 1, ConfigDef.Width.MEDIUM, HBASE_CONFIG_DIR_DISPLAY)

    //security
    .define(KerberosKey, Type.BOOLEAN, KerberosDefault, Importance.MEDIUM, KerberosDoc, "Security", 1, ConfigDef.Width.MEDIUM, KerberosDisplay)
    .define(KerberosAuthModeKey, Type.STRING, KerberosAuthModeDefault, Importance.MEDIUM, KerberosAuthModeDoc, "Security", 2, ConfigDef.Width.MEDIUM, KerberosAuthModeDisplay)
    .define(KerberosDebugKey, Type.BOOLEAN, KerberosDebugDefault, Importance.MEDIUM, KerberosDebugDoc, "Security", 3, ConfigDef.Width.MEDIUM, KerberosDebugDisplay)

    //keytab
    .define(PrincipalKey, Type.STRING, PrincipalDefault, Importance.MEDIUM, PrincipalDoc, "Kerberos Keytab", 1, ConfigDef.Width.MEDIUM, PrincipalDisplay)
    .define(KerberosKeyTabKey, Type.STRING, KerberosKeyTabDefault, Importance.MEDIUM, KerberosKeyTabDoc, "Kerberos Keytab", 2, ConfigDef.Width.MEDIUM, KerberosKeyTabDisplay)
    .define(NameNodePrincipalKey, Type.STRING, NameNodePrincipalDefault, Importance.MEDIUM, NameNodePrincipalDoc, "Kerberos Keytab", 3, ConfigDef.Width.MEDIUM, NameNodePrincipalDisplay)
    .define(KerberosTicketRenewalKey, Type.LONG, KerberosTicketRenewalDefault, Importance.MEDIUM, KerberosTicketRenewalDoc, "Kerberos Keytab", 4, ConfigDef.Width.MEDIUM, KerberosTicketRenewalDisplay)
    
    //user password
    .define(KerberosUserKey, Type.STRING, KerberosUserDefault, Importance.MEDIUM, KerberosUserDoc, "Kerberos User Password", 1, ConfigDef.Width.MEDIUM, KerberosUserDisplay)
    .define(KerberosPasswordKey, Type.PASSWORD, KerberosPasswordDefault, Importance.MEDIUM, KerberosPasswordDoc, "Kerberos User Password", 2, ConfigDef.Width.MEDIUM, KerberosPasswordDisplay)
    .define(KerberosKrb5Key, Type.STRING, KerberosKrb5Default, Importance.MEDIUM, KerberosKrb5Doc, "Kerberos User Password", 3, ConfigDef.Width.MEDIUM, KerberosKrb5Display)
    .define(KerberosJaasKey, Type.STRING, KerberosJaasDefault, Importance.MEDIUM, KerberosJaasDoc, "Kerberos User Password", 4, ConfigDef.Width.MEDIUM, KerberosJaasDisplay)
    .define(JaasEntryNameKey, Type.STRING, JaasEntryNameDefault, Importance.MEDIUM, JaasEntryNameDoc, "Kerberos User Password", 5, ConfigDef.Width.MEDIUM, JaasEntryNameDisplay)

}


case class HBaseConfig(props: util.Map[String, String])
  extends BaseConfig(CONNECTOR_PREFIX, HBaseConfig.config, props)
    with KcqlSettings
    with ErrorPolicySettings
    with NumberRetriesSettings

Source File: InfluxConfig.scala From stream-reactor with Apache License 2.0

5 votes

package com.datamountaineer.streamreactor.connect.influx.config

import java.util

import com.datamountaineer.streamreactor.connect.config.base.traits._
import org.apache.kafka.common.config.ConfigDef
import org.apache.kafka.common.config.ConfigDef.{Importance, Type}
import org.influxdb.InfluxDB.ConsistencyLevel

object InfluxConfig {

  val config: ConfigDef = new ConfigDef()
    .define(InfluxConfigConstants.INFLUX_URL_CONFIG, Type.STRING, Importance.HIGH, InfluxConfigConstants.INFLUX_URL_DOC, "Connection", 1, ConfigDef.Width.MEDIUM, InfluxConfigConstants.INFLUX_URL_CONFIG)
    .define(InfluxConfigConstants.INFLUX_DATABASE_CONFIG, Type.STRING, Importance.HIGH, InfluxConfigConstants.INFLUX_DATABASE_DOC, "Connection", 2, ConfigDef.Width.MEDIUM, InfluxConfigConstants.INFLUX_DATABASE_CONFIG)
    .define(InfluxConfigConstants.INFLUX_CONNECTION_USER_CONFIG, Type.STRING, Importance.HIGH, InfluxConfigConstants.INFLUX_CONNECTION_USER_DOC, "Connection", 3, ConfigDef.Width.MEDIUM, InfluxConfigConstants.INFLUX_CONNECTION_USER_CONFIG)
    .define(InfluxConfigConstants.INFLUX_CONNECTION_PASSWORD_CONFIG, Type.PASSWORD, "", Importance.HIGH, InfluxConfigConstants.INFLUX_CONNECTION_PASSWORD_DOC, "Connection", 4, ConfigDef.Width.MEDIUM, InfluxConfigConstants.INFLUX_CONNECTION_PASSWORD_CONFIG)
    .define(InfluxConfigConstants.KCQL_CONFIG, Type.STRING, Importance.HIGH, InfluxConfigConstants.KCQL_DOC, "Connection", 5, ConfigDef.Width.MEDIUM, InfluxConfigConstants.KCQL_DISPLAY)
    .define(InfluxConfigConstants.ERROR_POLICY_CONFIG, Type.STRING, InfluxConfigConstants.ERROR_POLICY_DEFAULT, Importance.HIGH, InfluxConfigConstants.ERROR_POLICY_DOC, "Miscellaneous", 1, ConfigDef.Width.MEDIUM, InfluxConfigConstants.ERROR_POLICY_CONFIG)
    .define(InfluxConfigConstants.ERROR_RETRY_INTERVAL_CONFIG, Type.INT, InfluxConfigConstants.ERROR_RETRY_INTERVAL_DEFAULT, Importance.MEDIUM, InfluxConfigConstants.ERROR_RETRY_INTERVAL_DOC, "Miscellaneous", 2, ConfigDef.Width.MEDIUM, InfluxConfigConstants.ERROR_RETRY_INTERVAL_CONFIG)
    .define(InfluxConfigConstants.NBR_OF_RETRIES_CONFIG, Type.INT, InfluxConfigConstants.NBR_OF_RETIRES_DEFAULT, Importance.MEDIUM, InfluxConfigConstants.NBR_OF_RETRIES_DOC, "Miscellaneous", 3, ConfigDef.Width.MEDIUM, InfluxConfigConstants.NBR_OF_RETRIES_CONFIG)
    .define(InfluxConfigConstants.RETENTION_POLICY_CONFIG, Type.STRING, InfluxConfigConstants.RETENTION_POLICY_DEFAULT, Importance.HIGH, InfluxConfigConstants.RETENTION_POLICY_DOC, "Writes", 1, ConfigDef.Width.MEDIUM, InfluxConfigConstants.RETENTION_POLICY_DOC)
    .define(InfluxConfigConstants.CONSISTENCY_CONFIG, Type.STRING, InfluxConfigConstants.CONSISTENCY_DEFAULT, Importance.MEDIUM, InfluxConfigConstants.CONSISTENCY_DOC, "Writes", 2, ConfigDef.Width.MEDIUM, InfluxConfigConstants.CONSISTENCY_DISPLAY)
    .define(InfluxConfigConstants.PROGRESS_COUNTER_ENABLED, Type.BOOLEAN, InfluxConfigConstants.PROGRESS_COUNTER_ENABLED_DEFAULT, Importance.MEDIUM, InfluxConfigConstants.PROGRESS_COUNTER_ENABLED_DOC, "Metrics", 1, ConfigDef.Width.MEDIUM, InfluxConfigConstants.PROGRESS_COUNTER_ENABLED_DISPLAY)
}


case class InfluxConfig(props: util.Map[String, String])
  extends BaseConfig(InfluxConfigConstants.CONNECTOR_PREFIX, InfluxConfig.config, props)
    with KcqlSettings
    with ErrorPolicySettings
    with NumberRetriesSettings
    with DatabaseSettings
    with ConsistencyLevelSettings[ConsistencyLevel]
    with UserSettings

Source File: HiveSinkConnector.scala From stream-reactor with Apache License 2.0

5 votes

package com.landoop.streamreactor.connect.hive.sink

import java.util

import com.datamountaineer.streamreactor.connect.utils.JarManifest
import com.landoop.streamreactor.connect.hive.sink.config.HiveSinkConfigDef
import org.apache.kafka.common.config.ConfigDef
import org.apache.kafka.connect.connector.Task
import org.apache.kafka.connect.sink.SinkConnector

import scala.collection.JavaConverters._

class HiveSinkConnector extends SinkConnector {

  val logger = org.slf4j.LoggerFactory.getLogger(getClass.getName)

  private val manifest = JarManifest(getClass.getProtectionDomain.getCodeSource.getLocation)
  private var props: util.Map[String, String] = _

  override def version(): String = manifest.version()
  override def taskClass(): Class[_ <: Task] = classOf[HiveSinkTask]
  override def config(): ConfigDef = HiveSinkConfigDef.config

  override def start(props: util.Map[String, String]): Unit = {
    logger.info(s"Creating hive sink connector")
    this.props = props
  }

  override def stop(): Unit = ()

  override def taskConfigs(maxTasks: Int): util.List[util.Map[String, String]] = {
    logger.info(s"Creating $maxTasks tasks config")
    List.fill(maxTasks)(props).asJava
  }
}

Source File: HiveSinkConfigDef.scala From stream-reactor with Apache License 2.0

5 votes

package com.landoop.streamreactor.connect.hive.sink.config

import java.util

import com.datamountaineer.streamreactor.connect.config.base.traits._
import org.apache.kafka.common.config.ConfigDef
import org.apache.kafka.common.config.ConfigDef.{Importance, Type}

object HiveSinkConfigDef {

  import SinkConfigSettings._

  val config: ConfigDef = new ConfigDef()
    .define(PROGRESS_COUNTER_ENABLED, Type.BOOLEAN, PROGRESS_COUNTER_ENABLED_DEFAULT,
      Importance.MEDIUM, PROGRESS_COUNTER_ENABLED_DOC,
      "Metrics", 1, ConfigDef.Width.MEDIUM, PROGRESS_COUNTER_ENABLED_DISPLAY)
    .define(KcqlKey, Type.STRING, Importance.HIGH, KCQL_DOC)
    .define(DatabaseNameKey, Type.STRING, Importance.HIGH, DatabaseNameDoc)
    .define(MetastoreTypeKey, Type.STRING, Importance.HIGH, MetastoreTypeDoc)
    .define(MetastoreUrisKey, Type.STRING, Importance.HIGH, MetastoreUrisDoc)
    .define(FsDefaultKey, Type.STRING, Importance.HIGH, FsDefaultDoc)

    //config folders
    .define(HdfsConfigDirKey, Type.STRING, HdfsConfigDirDefault, Importance.MEDIUM, HdfsConfigDirDoc, "Configs", 1, ConfigDef.Width.MEDIUM, HdfsConfigDirDisplay)
    .define(HiveConfigDirKey, Type.STRING, HiveConfigDirDefault, Importance.MEDIUM, HiveConfigDirDoc, "Configs", 2, ConfigDef.Width.MEDIUM, HiveConfigDirDisplay)

    //security
    .define(KerberosKey, Type.BOOLEAN, KerberosDefault, Importance.MEDIUM, KerberosDoc, "Security", 1, ConfigDef.Width.MEDIUM, KerberosDisplay)
    .define(KerberosAuthModeKey, Type.STRING, KerberosAuthModeDefault, Importance.MEDIUM, KerberosAuthModeDoc, "Security", 2, ConfigDef.Width.MEDIUM, KerberosAuthModeDisplay)


    //keytab
    .define(PrincipalKey, Type.STRING, PrincipalDefault, Importance.MEDIUM, PrincipalDoc, "Kerberos Keytab", 1, ConfigDef.Width.MEDIUM, PrincipalDisplay)
    .define(KerberosKeyTabKey, Type.STRING, KerberosKeyTabDefault, Importance.MEDIUM, KerberosKeyTabDoc, "Kerberos Keytab", 2, ConfigDef.Width.MEDIUM, KerberosKeyTabDisplay)
    .define(NameNodePrincipalKey, Type.STRING, NameNodePrincipalDefault, Importance.MEDIUM, NameNodePrincipalDoc, "Kerberos Keytab", 3, ConfigDef.Width.MEDIUM, NameNodePrincipalDisplay)
    .define(KerberosTicketRenewalKey, Type.LONG, KerberosTicketRenewalDefault, Importance.MEDIUM, KerberosTicketRenewalDoc, "Kerberos Keytab", 4, ConfigDef.Width.MEDIUM, KerberosTicketRenewalDisplay)


    //user password
    .define(KerberosUserKey, Type.STRING, KerberosUserDefault, Importance.MEDIUM, KerberosUserDoc, "Kerberos User Password", 1, ConfigDef.Width.MEDIUM, KerberosUserDisplay)
    .define(KerberosPasswordKey, Type.PASSWORD, KerberosPasswordDefault, Importance.MEDIUM, KerberosPasswordDoc, "Kerberos User Password", 2, ConfigDef.Width.MEDIUM, KerberosPasswordDisplay)
    .define(KerberosKrb5Key, Type.STRING, KerberosKrb5Default, Importance.MEDIUM, KerberosKrb5Doc, "Kerberos User Password", 3, ConfigDef.Width.MEDIUM, KerberosKrb5Display)
    .define(KerberosJaasKey, Type.STRING, KerberosJaasDefault, Importance.MEDIUM, KerberosJaasDoc, "Kerberos User Password", 4, ConfigDef.Width.MEDIUM, KerberosJaasDisplay)
    .define(JaasEntryNameKey, Type.STRING, JaasEntryNameDefault, Importance.MEDIUM, JaasEntryNameDoc, "Kerberos User Password", 5, ConfigDef.Width.MEDIUM, JaasEntryNameDisplay)
}

case class HiveSinkConfigDefBuilder(props: util.Map[String, String])
  extends BaseConfig(SinkConfigSettings.CONNECTOR_PREFIX, HiveSinkConfigDef.config, props)
    with KcqlSettings
    with ErrorPolicySettings
    with NumberRetriesSettings
    with UserSettings
    with ConnectionSettings

Source File: HiveSourceConnector.scala From stream-reactor with Apache License 2.0

5 votes

package com.landoop.streamreactor.connect.hive.source

import java.util

import com.datamountaineer.streamreactor.connect.utils.JarManifest
import com.landoop.streamreactor.connect.hive.sink.config.HiveSinkConfigDef
import com.typesafe.scalalogging.StrictLogging
import org.apache.kafka.common.config.ConfigDef
import org.apache.kafka.connect.connector.Task
import org.apache.kafka.connect.source.SourceConnector

import scala.collection.JavaConverters._

class HiveSourceConnector extends SourceConnector with StrictLogging {

  private val manifest = JarManifest(getClass.getProtectionDomain.getCodeSource.getLocation)
  private var props: util.Map[String, String] = _

  override def version(): String = manifest.version()
  override def taskClass(): Class[_ <: Task] = classOf[HiveSourceTask]
  override def config(): ConfigDef = HiveSinkConfigDef.config

  override def start(props: util.Map[String, String]): Unit = {
    logger.info(s"Creating hive sink connector")
    this.props = props
  }

  override def stop(): Unit = ()

  override def taskConfigs(maxTasks: Int): util.List[util.Map[String, String]] = {
    logger.info(s"Creating $maxTasks tasks config")
    List.fill(maxTasks)(props).asJava
  }
}

Source File: HiveSourceConfigDef.scala From stream-reactor with Apache License 2.0

5 votes

package com.landoop.streamreactor.connect.hive.source.config

import java.util

import com.datamountaineer.streamreactor.connect.config.base.traits._
import org.apache.kafka.common.config.ConfigDef
import org.apache.kafka.common.config.ConfigDef.{Importance, Type}

object HiveSourceConfigDef {

  import HiveSourceConfigConstants._

  val config: ConfigDef = new ConfigDef()
    .define(PROGRESS_COUNTER_ENABLED, Type.BOOLEAN, PROGRESS_COUNTER_ENABLED_DEFAULT,
      Importance.MEDIUM, PROGRESS_COUNTER_ENABLED_DOC,
      "Metrics", 1, ConfigDef.Width.MEDIUM, PROGRESS_COUNTER_ENABLED_DISPLAY)
    .define(KcqlKey, Type.STRING, Importance.HIGH, KCQL_DOC)
    .define(DatabaseNameKey, Type.STRING, Importance.HIGH, DatabaseNameDoc)
    .define(MetastoreTypeKey, Type.STRING, Importance.HIGH, MetastoreTypeDoc)
    .define(MetastoreUrisKey, Type.STRING, Importance.HIGH, MetastoreUrisDoc)
    .define(FsDefaultKey, Type.STRING, Importance.HIGH, FsDefaultDoc)
    .define(PollSizeKey, Type.INT, 1024, Importance.HIGH, PollSizeDoc)

    //config folders
    .define(HdfsConfigDirKey, Type.STRING, HdfsConfigDirDefault, Importance.MEDIUM, HdfsConfigDirDoc, "Configs", 1, ConfigDef.Width.MEDIUM, HdfsConfigDirDisplay)
    .define(HiveConfigDirKey, Type.STRING, HiveConfigDirDefault, Importance.MEDIUM, HiveConfigDirDoc, "Configs", 2, ConfigDef.Width.MEDIUM, HiveConfigDirDisplay)

    //security
    .define(KerberosKey, Type.BOOLEAN, KerberosDefault, Importance.MEDIUM, KerberosDoc, "Security", 1, ConfigDef.Width.MEDIUM, KerberosDisplay)
    .define(KerberosAuthModeKey, Type.STRING, KerberosAuthModeDefault, Importance.MEDIUM, KerberosAuthModeDoc, "Security", 2, ConfigDef.Width.MEDIUM, KerberosAuthModeDisplay)


    //keytab
    .define(PrincipalKey, Type.STRING, PrincipalDefault, Importance.MEDIUM, PrincipalDoc, "Kerberos Keytab", 1, ConfigDef.Width.MEDIUM, PrincipalDisplay)
    .define(KerberosKeyTabKey, Type.STRING, KerberosKeyTabDefault, Importance.MEDIUM, KerberosKeyTabDoc, "Kerberos Keytab", 2, ConfigDef.Width.MEDIUM, KerberosKeyTabDisplay)
    .define(NameNodePrincipalKey, Type.STRING, NameNodePrincipalDefault, Importance.MEDIUM, NameNodePrincipalDoc, "Kerberos Keytab", 3, ConfigDef.Width.MEDIUM, NameNodePrincipalDisplay)
    .define(KerberosTicketRenewalKey, Type.LONG, KerberosTicketRenewalDefault, Importance.MEDIUM, KerberosTicketRenewalDoc, "Kerberos Keytab", 4, ConfigDef.Width.MEDIUM, KerberosTicketRenewalDisplay)


    //user password
    .define(KerberosUserKey, Type.STRING, KerberosUserDefault, Importance.MEDIUM, KerberosUserDoc, "Kerberos User Password", 1, ConfigDef.Width.MEDIUM, KerberosUserDisplay)
    .define(KerberosPasswordKey, Type.PASSWORD, KerberosPasswordDefault, Importance.MEDIUM, KerberosPasswordDoc, "Kerberos User Password", 2, ConfigDef.Width.MEDIUM, KerberosPasswordDisplay)
    .define(KerberosKrb5Key, Type.STRING, KerberosKrb5Default, Importance.MEDIUM, KerberosKrb5Doc, "Kerberos User Password", 3, ConfigDef.Width.MEDIUM, KerberosKrb5Display)
    .define(KerberosJaasKey, Type.STRING, KerberosJaasDefault, Importance.MEDIUM, KerberosJaasDoc, "Kerberos User Password", 4, ConfigDef.Width.MEDIUM, KerberosJaasDisplay)
    .define(JaasEntryNameKey, Type.STRING, JaasEntryNameDefault, Importance.MEDIUM, JaasEntryNameDoc, "Kerberos User Password", 5, ConfigDef.Width.MEDIUM, JaasEntryNameDisplay)
}

case class HiveSourceConfigDefBuilder(props: util.Map[String, String])
  extends BaseConfig(HiveSourceConfigConstants.CONNECTOR_PREFIX, HiveSourceConfigDef.config, props)
    with KcqlSettings
    with ErrorPolicySettings
    with NumberRetriesSettings
    with UserSettings
    with ConnectionSettings

Source File: BloombergSourceConfig.scala From stream-reactor with Apache License 2.0

5 votes

package com.datamountaineer.streamreactor.connect.bloomberg.config

import org.apache.kafka.common.config.ConfigDef.{Importance, Type}
import org.apache.kafka.common.config.{AbstractConfig, ConfigDef}


class BloombergSourceConfig(map: java.util.Map[String, String]) extends AbstractConfig(BloombergSourceConfig.config, map)

object BloombergSourceConfig {


  lazy val config: ConfigDef = new ConfigDef()
    .define(BloombergSourceConfigConstants.SERVER_HOST, Type.STRING, Importance.HIGH, BloombergSourceConfigConstants.SERVER_HOST_DOC, "Connection", 1, ConfigDef.Width.MEDIUM, BloombergSourceConfigConstants.SERVER_HOST)
    .define(BloombergSourceConfigConstants.SERVER_PORT, Type.INT, Importance.HIGH, BloombergSourceConfigConstants.SERVER_PORT_DOC, "Connection", 2, ConfigDef.Width.MEDIUM, BloombergSourceConfigConstants.SERVER_PORT)
    .define(BloombergSourceConfigConstants.SERVICE_URI, Type.STRING, Importance.HIGH, BloombergSourceConfigConstants.SERVICE_URI_DOC,  "Connection", 3, ConfigDef.Width.MEDIUM, BloombergSourceConfigConstants.SERVICE_URI)
    .define(BloombergSourceConfigConstants.SUBSCRIPTIONS, Type.STRING, Importance.HIGH, BloombergSourceConfigConstants.SUBSCRIPTION_DOC,  "Subscription", 1, ConfigDef.Width.MEDIUM, BloombergSourceConfigConstants.SUBSCRIPTIONS)
    .define(BloombergSourceConfigConstants.AUTHENTICATION_MODE, Type.STRING, Importance.LOW, BloombergSourceConfigConstants.AUTHENTICATION_MODE_DOC, "Connection", 4, ConfigDef.Width.MEDIUM, BloombergSourceConfigConstants.AUTHENTICATION_MODE)
    .define(BloombergSourceConfigConstants.KAFKA_TOPIC, Type.STRING, Importance.HIGH, BloombergSourceConfigConstants.KAFKA_TOPIC_DOC, "Subscription", 2, ConfigDef.Width.MEDIUM, BloombergSourceConfigConstants.KAFKA_TOPIC)
    .define(BloombergSourceConfigConstants.BUFFER_SIZE, Type.INT, Importance.MEDIUM, BloombergSourceConfigConstants.BUFFER_SIZE_DOC, "Connection", 5, ConfigDef.Width.SHORT, BloombergSourceConfigConstants.BUFFER_SIZE)
    .define(BloombergSourceConfigConstants.PAYLOAD_TYPE, Type.STRING, Importance.MEDIUM, BloombergSourceConfigConstants.PAYLOAD_TYPE_DOC, "Subscription", 3, ConfigDef.Width.SHORT, BloombergSourceConfigConstants.PAYLOAD_TYPE)
    .define(BloombergSourceConfigConstants.PROGRESS_COUNTER_ENABLED, Type.BOOLEAN, BloombergSourceConfigConstants.PROGRESS_COUNTER_ENABLED_DEFAULT, Importance.MEDIUM, BloombergSourceConfigConstants.PROGRESS_COUNTER_ENABLED_DOC, "Metrics", 1, ConfigDef.Width.MEDIUM, BloombergSourceConfigConstants.PROGRESS_COUNTER_ENABLED_DISPLAY)
    .define(BloombergSourceConfigConstants.SERVICE_AUTHORIZATION, Type.BOOLEAN, Importance.LOW, BloombergSourceConfigConstants.SERVICE_AUTHORIZATION_DOC,  "Connection", 6, ConfigDef.Width.MEDIUM, BloombergSourceConfigConstants.SERVICE_AUTHORIZATION)
}

Source File: ReThinkSinkConnector.scala From stream-reactor with Apache License 2.0

5 votes

package com.datamountaineer.streamreactor.connect.rethink.sink

import java.util

import com.datamountaineer.streamreactor.connect.config.Helpers
import com.datamountaineer.streamreactor.connect.rethink.ReThinkConnection
import com.datamountaineer.streamreactor.connect.rethink.config.{ReThinkConfigConstants, ReThinkSinkConfig, ReThinkSinkSettings}
import com.datamountaineer.streamreactor.connect.utils.JarManifest
import com.rethinkdb.RethinkDB
import com.typesafe.scalalogging.StrictLogging
import org.apache.kafka.common.config.ConfigDef
import org.apache.kafka.connect.connector.Task
import org.apache.kafka.connect.sink.SinkConnector

import scala.collection.JavaConverters._



    val rethink = RethinkDB.r
    initializeTables(rethink, props)
    configProps = props
  }

  def initializeTables(rethink: RethinkDB, props: util.Map[String, String]): Unit = {
    val config = ReThinkSinkConfig(props)
    val settings = ReThinkSinkSettings(config)
    val rethinkHost = config.getString(ReThinkConfigConstants.RETHINK_HOST)
    val port = config.getInt(ReThinkConfigConstants.RETHINK_PORT)

    val conn = ReThinkConnection(rethink, config)
    ReThinkHelper.checkAndCreateTables(rethink, settings, conn)
    conn.close()
  }

  override def stop(): Unit = {}

  override def version(): String = manifest.version()

  override def config(): ConfigDef = configDef
}

org.apache.kafka.common.config.ConfigDef Scala Examples