Java Code Examples for org.apache.kafka.common.config.ConfigDef#define()

public KafkaNodeClient(int id, String host, int port) {
	node = new Node(id, host, port);
	
	//
	LogContext logContext = new LogContext("ctx");

	ConfigDef defConf = new ConfigDef();
	defConf.define(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, ConfigDef.Type.STRING,
			CommonClientConfigs.DEFAULT_SECURITY_PROTOCOL, ConfigDef.Importance.MEDIUM,
			CommonClientConfigs.SECURITY_PROTOCOL_DOC);

	defConf.define(SaslConfigs.SASL_MECHANISM, ConfigDef.Type.STRING, SaslConfigs.DEFAULT_SASL_MECHANISM,
			ConfigDef.Importance.MEDIUM, SaslConfigs.SASL_MECHANISM_DOC);

	metrics = new Metrics(Time.SYSTEM);

	AbstractConfig config = new AbstractConfig(defConf, new Properties());
	channelBuilder = ClientUtils.createChannelBuilder(config);
	selector = new Selector(1000L, metrics, Time.SYSTEM, "cc", channelBuilder, logContext);
	client = new NetworkClient(selector, new Metadata(0, Long.MAX_VALUE, false),
			CLIENT_ID, 10, 1000L, 1000L, 1, 1024, 1000, Time.SYSTEM, true, new ApiVersions(),
			null, logContext);
}
 

@Test
public void testNullDefaultWithValidator() {
    final String key = "enum_test";

    ConfigDef def = new ConfigDef();
    def.define(key, Type.STRING, ConfigDef.NO_DEFAULT_VALUE,
               ValidString.in("ONE", "TWO", "THREE"), Importance.HIGH, "docs");

    Properties props = new Properties();
    props.put(key, "ONE");
    Map<String, Object> vals = def.parse(props);
    assertEquals("ONE", vals.get(key));
}
 

public static void clickHouseConfigDef(ConfigDef config) {
    config.define(TOPICS, ConfigDef.Type.STRING,
            ConfigDef.Importance.HIGH, "消费的Kafka topic列表，以逗号分隔");
    config.define(CLICKHOUSE_HOSTS, ConfigDef.Type.STRING,
            ConfigDef.Importance.HIGH, "连接clickhouse的host列表，以逗号分隔");
    config.define(CLICKHOUSE_JDBC_PORT, ConfigDef.Type.STRING,
            ConfigDef.Importance.HIGH, "连接clickhouse jdbc端口号");
    config.define(CLICKHOUSE_JDBC_USER, ConfigDef.Type.STRING, "",
            ConfigDef.Importance.LOW, "连接clickhouse jdbc user");
    config.define(CLICKHOUSE_JDBC_PASSWORD, ConfigDef.Type.STRING, "",
            ConfigDef.Importance.LOW, "连接clickhouse jdbc password");
    config.define(CLICKHOUSE_SINK_DATABASE, ConfigDef.Type.STRING,
            ConfigDef.Importance.HIGH, "sink 至clickhouse 的数据库");
}
 

public static void clickHousePartitionConfigDef(ConfigDef config) {
    config.define(CLICKHOUSE_SINK_DATE_COLUMNS, ConfigDef.Type.STRING, "",
            ConfigDef.Importance.HIGH, "写到ClickHouse表中的日期分区字段名, 多个以逗号分隔，不可为空" +
                    " 其值应与ClickHouse对应分布式表一一对应. 即: 需要与clickhouse.sink.tables的参数一一对应. ");
    config.define(CLICKHOUSE_SOURCE_DATE_COLUMNS, ConfigDef.Type.STRING, "",
            ConfigDef.Importance.LOW, "写到ClickHouse表中的时间分区字段所取值的源字段名" +
                    " 1.当此参数为空时, 将默认取当前处理的时间写至clickhouse.sink.date.columns所描述的字段中" +
                    " 2.当此参数不为空时, 其值应与ClickHouse对应分布式表一一对应. 即: 需要与clickhouse.sink.tables的参数一一对应. ");
    config.define(CLICKHOUSE_SOURCE_DATE_FORMAT, ConfigDef.Type.STRING, "yyyy-MM-dd HH:mm:ss",
            ConfigDef.Importance.LOW, "写到ClickHouse表中的时间分区字段所取值的源字段时间的格式" +
                    " 1.当此参数为空时, 将默认yyyy-MM-dd HH:mm:ss格式化" +
                    " 2.当此参数不为空时, 其值应与ClickHouse对应分布式表一一对应. 即: 需要与clickhouse.sink.tables的参数一一对应. ");
}
 

private static ConfigDef getConfig() {
  Map<String, ConfigDef.ConfigKey> everything = new HashMap<>(conf().configKeys());
  ConfigDef visible = new ConfigDef();
  for (ConfigDef.ConfigKey key : everything.values()) {
    visible.define(key);
  }
  return visible;
}
 

public static ConfigDef conf(Map<String, ?> unparsedConfig) {
  String group = "REST_HTTP";
  int orderInGroup = 0;
  ConfigDef config = new ConfigDef()
    .define(RESPONSE_VAR_NAMES_CONFIG,
      Type.LIST,
      RESPONSE_VAR_NAMES_DEFAULT,
      Importance.LOW,
      RESPONSE_VAR_NAMES_DOC,
      group,
      ++orderInGroup,
      ConfigDef.Width.SHORT,
      RESPONSE_VAR_NAMES_DISPLAY)
    ;

  // This is a bit hacky and there may be a better way of doing it, but I don't know it.
  // We need to create config items dynamically, based on the parameter names,
  // so we need a 2 pass parse of the config.
  List<String> varNames = (List) config.parse(unparsedConfig).get(RESPONSE_VAR_NAMES_CONFIG);

  for(String varName : varNames) {
    config.define(String.format(RESPONSE_VAR_XPATH_CONFIG, varName),
      Type.STRING,
      RESPONSE_VAR_XPATH_DEFAULT,
      Importance.HIGH,
      String.format(RESPONSE_VAR_XPATH_DOC, varName),
      group,
      ++orderInGroup,
      ConfigDef.Width.SHORT,
      String.format(RESPONSE_VAR_XPATH_DISPLAY, varName));
  }

  return(config);
}
 

public static ConfigDef conf(Map<String, ?> unparsedConfig) {
  String group = "REST_HTTP";
  int orderInGroup = 0;
  ConfigDef config = new ConfigDef()
    .define(RESPONSE_VAR_NAMES_CONFIG,
      Type.LIST,
      RESPONSE_VAR_NAMES_DEFAULT,
      Importance.LOW,
      RESPONSE_VAR_NAMES_DOC,
      group,
      ++orderInGroup,
      ConfigDef.Width.SHORT,
      RESPONSE_VAR_NAMES_DISPLAY)
    ;

  // This is a bit hacky and there may be a better way of doing it, but I don't know it.
  // We need to create config items dynamically, based on the parameter names,
  // so we need a 2 pass parse of the config.
  List<String> varNames = (List) config.parse(unparsedConfig).get(RESPONSE_VAR_NAMES_CONFIG);

  for(String varName : varNames) {
    config.define(String.format(RESPONSE_VAR_REGEX_CONFIG, varName),
      Type.STRING,
      RESPONSE_VAR_REGEX_DEFAULT,
      Importance.HIGH,
      String.format(RESPONSE_VAR_REGEX_DOC, varName),
      group,
      ++orderInGroup,
      ConfigDef.Width.SHORT,
      String.format(RESPONSE_VAR_REGEX_DISPLAY, varName));
  }

  return(config);
}
 

private static ConfigDef getConfig() {
  Map<String, ConfigDef.ConfigKey> everything = new HashMap<>(conf().configKeys());
  ConfigDef visible = new ConfigDef();
  for (ConfigDef.ConfigKey key : everything.values()) {
    visible.define(key);
  }
  return visible;
}
 

/**
 * Returns the given Kafka ConfigDef augmented with config keys from
 * the given interface.
 */
public <T> ConfigDef define(Class<T> configInterface, ConfigDef def) {
  for (Method method : configInterface.getMethods()) {
    if (Modifier.isStatic(method.getModifiers())) {
      continue;
    }

    validateReturnType(method);

    def.define(new ConfigDef.ConfigKey(
        getConfigKeyName(method),
        getKafkaType(method),
        getDefaultValue(method),
        getValidator(method),
        getImportance(method),
        getDocumentation(method),
        getGroup(method),
        getOrderInGroup(method),
        getWidth(method),
        getDisplayName(method),
        getDependents(method),
        getRecommender(method),
        false));
  }

  return def;
}
 

@Override
public ConfigDef config() {
    ConfigDef defs = new ConfigDef();
    defs.define("influxdb.url", ConfigDef.Type.STRING, "http://localhost:8086", ConfigDef.Importance.HIGH, "influxdb server http address in the form http://<host>:<port>");
    defs.define("influxdb.database", ConfigDef.Type.STRING, "metrics", ConfigDef.Importance.HIGH, "influxdb database name to which to publish");
    defs.define("influxdb.username", ConfigDef.Type.STRING, "", ConfigDef.Importance.MEDIUM, "influxdb username to use for http updates");
    defs.define("influxdb.password", ConfigDef.Type.STRING, "", ConfigDef.Importance.MEDIUM, "influxdb password to use for http updates");
    return defs;
}
 

private void setServoReporter() {
    props.put("metric.reporters", Lists.newArrayList(ServoReporter.class.getName()));
    // this should be needed because ProducerConfig cannot retrieve undefined key
    try {
        Field f = ProducerConfig.class.getDeclaredField("config");
        f.setAccessible(true);
        ConfigDef config = (ConfigDef) f.get(ConfigDef.class);
        config.define(ServoReporter.class.getName(), ConfigDef.Type.CLASS, ServoReporter.class, ConfigDef.Importance.LOW, "");
    } catch (Exception e) {
        // swallow exception
    }
    props.put(ServoReporter.class.getName(), ServoReporter.class);
}
 

private static void addGcsConfigGroup(final ConfigDef configDef) {
    int gcsGroupCounter = 0;
    configDef.define(
        GCS_CREDENTIALS_PATH_CONFIG,
        ConfigDef.Type.STRING,
        null,
        ConfigDef.Importance.LOW,
        "The path to a GCP credentials file. "
            + "If not provided, the connector will try to detect the credentials automatically. "
            + "Cannot be set together with \"" + GCS_CREDENTIALS_JSON_CONFIG + "\"",
        GROUP_GCS,
        gcsGroupCounter++,
        ConfigDef.Width.NONE,
        GCS_CREDENTIALS_PATH_CONFIG
    );

    configDef.define(
        GCS_CREDENTIALS_JSON_CONFIG,
        ConfigDef.Type.PASSWORD,
        null,
        ConfigDef.Importance.LOW,
        "GCP credentials as a JSON string. "
            + "If not provided, the connector will try to detect the credentials automatically. "
            + "Cannot be set together with \"" + GCS_CREDENTIALS_PATH_CONFIG + "\"",
        GROUP_GCS,
        gcsGroupCounter++,
        ConfigDef.Width.NONE,
        GCS_CREDENTIALS_JSON_CONFIG
    );

    configDef.define(
        GCS_BUCKET_NAME_CONFIG,
        ConfigDef.Type.STRING,
        ConfigDef.NO_DEFAULT_VALUE,
        new ConfigDef.NonEmptyString(),
        ConfigDef.Importance.HIGH,
        "The GCS bucket name to store output files in.",
        GROUP_GCS,
        gcsGroupCounter++,
        ConfigDef.Width.NONE,
        GCS_BUCKET_NAME_CONFIG
    );
}
 

private static ConfigDef createConfigDef() {
  ConfigDef configDef =
      new ConfigDef() {

        @Override
        @SuppressWarnings("unchecked")
        public Map<String, ConfigValue> validateAll(final Map<String, String> props) {
          Map<String, ConfigValue> results = super.validateAll(props);
          // Don't validate child configs if the top level configs are broken
          if (results.values().stream().anyMatch((c) -> !c.errorMessages().isEmpty())) {
            return results;
          }

          boolean hasTopicsConfig = !props.getOrDefault(TOPICS_CONFIG, "").trim().isEmpty();
          boolean hasTopicsRegexConfig =
              !props.getOrDefault(TOPICS_REGEX_CONFIG, "").trim().isEmpty();

          if (hasTopicsConfig && hasTopicsRegexConfig) {
            results
                .get(TOPICS_CONFIG)
                .addErrorMessage(
                    format(
                        "%s and %s are mutually exclusive options, but both are set.",
                        TOPICS_CONFIG, TOPICS_REGEX_CONFIG));
          } else if (!hasTopicsConfig && !hasTopicsRegexConfig) {
            results
                .get(TOPICS_CONFIG)
                .addErrorMessage(
                    format("Must configure one of %s or %s", TOPICS_CONFIG, TOPICS_REGEX_CONFIG));
          }

          if (hasTopicsConfig) {
            List<String> topics = (List<String>) results.get(TOPICS_CONFIG).value();
            topics.forEach(
                topic -> results.putAll(MongoSinkTopicConfig.validateAll(topic, props)));
          } else if (hasTopicsRegexConfig) {
            results.putAll(MongoSinkTopicConfig.validateRegexAll(props));
          }
          return results;
        }
      };
  String group = "Connection";
  int orderInGroup = 0;
  configDef.define(
      TOPICS_CONFIG,
      Type.LIST,
      TOPICS_DEFAULT,
      Importance.HIGH,
      TOPICS_DOC,
      group,
      ++orderInGroup,
      Width.MEDIUM,
      TOPICS_DISPLAY);

  configDef.define(
      TOPICS_REGEX_CONFIG,
      Type.STRING,
      TOPICS_REGEX_DEFAULT,
      Validators.isAValidRegex(),
      Importance.HIGH,
      TOPICS_REGEX_DOC,
      group,
      ++orderInGroup,
      Width.MEDIUM,
      TOPICS_REGEX_DISPLAY);

  configDef.define(
      CONNECTION_URI_CONFIG,
      Type.STRING,
      CONNECTION_URI_DEFAULT,
      errorCheckingValueValidator("A valid connection string", ConnectionString::new),
      Importance.HIGH,
      CONNECTION_URI_DOC,
      group,
      ++orderInGroup,
      Width.MEDIUM,
      CONNECTION_URI_DISPLAY);

  group = "Overrides";
  orderInGroup = 0;
  configDef.define(
      TOPIC_OVERRIDE_CONFIG,
      Type.STRING,
      TOPIC_OVERRIDE_DEFAULT,
      Validators.topicOverrideValidator(),
      Importance.LOW,
      TOPIC_OVERRIDE_DOC,
      group,
      ++orderInGroup,
      Width.MEDIUM,
      TOPIC_OVERRIDE_DISPLAY);

  MongoSinkTopicConfig.BASE_CONFIG.configKeys().values().forEach(configDef::define);
  return configDef;
}
 

public static ConfigDef conf(Map<String, ?> unparsedConfig) {
  String group = "REST_HTTP";
  int orderInGroup = 0;
  ConfigDef config =  new ConfigDef()
    .define(REQUEST_BODY_CONFIG,
      Type.STRING,
      REQUEST_BODY_DEFAULT,
      Importance.LOW,
      REQUEST_BODY_DOC,
      group,
      ++orderInGroup,
      ConfigDef.Width.LONG,
      REQUEST_BODY_DISPLAY)

    .define(REQUEST_PARAMETER_NAMES_CONFIG,
      Type.LIST,
      REQUEST_PARAMETER_NAMES_DEFAULT,
      Importance.HIGH,
      REQUEST_PARAMETER_NAMES_DOC,
      group,
      ++orderInGroup,
      ConfigDef.Width.SHORT,
      REQUEST_PARAMETER_NAMES_DISPLAY)

    .define(REQUEST_HEADERS_CONFIG,
      Type.LIST,
      REQUEST_HEADERS_DEFAULT,
      Importance.HIGH,
      REQUEST_HEADERS_DOC,
      group,
      ++orderInGroup,
      ConfigDef.Width.SHORT,
      REQUEST_HEADERS_DISPLAY)
    ;

  // This is a bit hacky and there may be a better way of doing it, but I don't know it.
  // We need to create config items dynamically, based on the parameter names,
  // so we need a 2 pass parse of the config.
  List<String> paramNames = (List) config.parse(unparsedConfig).get(REQUEST_PARAMETER_NAMES_CONFIG);

  for(String paramName : paramNames) {
    config.define(String.format(REQUEST_PARAMETER_VALUE_CONFIG, paramName),
      Type.STRING,
      REQUEST_PARAMETER_VALUE_DEFAULT,
      Importance.HIGH,
      String.format(REQUEST_PARAMETER_VALUE_DOC, paramName),
      group,
      ++orderInGroup,
      ConfigDef.Width.SHORT,
      String.format(REQUEST_PARAMETER_VALUE_DISPLAY, paramName));
  }

  return(config);
}
 

public static ConfigDef conf(Map<String, ?> unparsedConfig) {
  String group = "REST_HTTP";
  int orderInGroup = 0;
  ConfigDef config = new ConfigDef()
    .define(VALUE_PROVIDER_CONFIG,
      Type.CLASS,
      VALUE_PROVIDER_DEFAULT,
      new InstanceOfValidator(ValueProvider.class),
      Importance.HIGH,
      VALUE_PROVIDER_DOC,
      group,
      ++orderInGroup,
      ConfigDef.Width.SHORT,
      VALUE_PROVIDER_DISPLAY,
      new ServiceProviderInterfaceRecommender(ValueProvider.class))

    .define(TEMPLATE_ENGINE_CONFIG,
      Type.CLASS,
      TEMPLATE_ENGINE_DEFAULT,
      new InstanceOfValidator(TemplateEngine.class),
      Importance.HIGH,
      TEMPLATE_ENGINE_DOC,
      group,
      ++orderInGroup,
      ConfigDef.Width.SHORT,
      TEMPLATE_ENGINE_DISPLAY,
      new ServiceProviderInterfaceRecommender(TemplateEngine.class))

    .define(REQUEST_BODY_TEMPLATE_CONFIG,
      Type.STRING,
      REQUEST_BODY_TEMPLATE_DEFAULT,
      Importance.LOW,
      REQUEST_BODY_TEMPLATE_DOC,
      group,
      ++orderInGroup,
      ConfigDef.Width.LONG,
      REQUEST_BODY_TEMPLATE_DISPLAY)

    .define(REQUEST_PARAMETER_NAMES_CONFIG,
      Type.LIST,
      REQUEST_PARAMETER_NAMES_DEFAULT,
      Importance.LOW,
      REQUEST_PARAMETER_NAMES_DOC,
      group,
      ++orderInGroup,
      ConfigDef.Width.SHORT,
      REQUEST_PARAMETER_NAMES_DISPLAY)

    .define(REQUEST_HEADERS_TEMPLATE_CONFIG,
      Type.LIST,
      REQUEST_HEADERS_TEMPLATE_DEFAULT,
      Importance.LOW,
      REQUEST_HEADERS_TEMPLATE_DOC,
      group,
      ++orderInGroup,
      ConfigDef.Width.SHORT,
      REQUEST_HEADERS_TEMPLATE_DISPLAY)
    ;

  // This is a bit hacky and there may be a better way of doing it, but I don't know it.
  // We need to create config items dynamically, based on the parameter names,
  // so we need a 2 pass parse of the config.
  List<String> paramNames = (List) config.parse(unparsedConfig).get(REQUEST_PARAMETER_NAMES_CONFIG);

  for(String paramName : paramNames) {
    config.define(String.format(REQUEST_PARAMETER_TEMPLATE_CONFIG, paramName),
      Type.STRING,
      REQUEST_PARAMETER_TEMPLATE_DEFAULT,
      Importance.HIGH,
      String.format(REQUEST_PARAMETER_TEMPLATE_DOC, paramName),
      group,
      ++orderInGroup,
      ConfigDef.Width.SHORT,
      String.format(REQUEST_PARAMETER_TEMPLATE_DISPLAY, paramName));
  }

  return(config);
}
 

/**
 * @param configDef - The configuration schema definition that will be updated to include
 *   this configuration's fields. (not null)
 */
public static void addCommonDefinitions(final ConfigDef configDef) {
    requireNonNull(configDef);
    configDef.define(RYA_INSTANCE_NAME, Type.STRING, Importance.HIGH, RYA_INSTANCE_NAME_DOC);
}