Java Code Examples for org.apache.kafka.common.config.ConfigValue#value()

The following examples show how to use org.apache.kafka.common.config.ConfigValue#value() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: MySqlSinkClickHouseConnector.java From kafka-connectors with Apache License 2.0

6 votes

vote down

vote up

/**
 * 对输入的本地表进行校验
 * 如果输入不为空：
 * *   如果输入的分布式表不为空, 则判断其是否与分布式表一一对应
 * *   如果输入的分布式表为空, 则判断其是否与topic个数一一对应
 */
private void localTableValidate(ConfigValue sinkTablesVal, String topics, ConfigValue sinkLocalTablesVal) {
    if (sinkLocalTablesVal.value() != null) {
        String sinkLocalTables = (String) sinkLocalTablesVal.value();
        String[] sinkLocalTabArr = sinkLocalTables.split(CONFIG_SEPARATOR);
        if (sinkTablesVal.value() != null) {
            String sinkTables = (String) sinkTablesVal.value();
            if (sinkLocalTabArr.length != sinkTables.split(CONFIG_SEPARATOR).length) {
                sinkLocalTablesVal.addErrorMessage(sinkLocalTablesVal.name() + "需要与" + sinkTablesVal.name() + "一一对应");
            }
        } else {
            String[] topicsArr = topics.split(CONFIG_SEPARATOR);
            if (topicsArr.length != sinkLocalTabArr.length) {
                sinkLocalTablesVal.addErrorMessage(sinkLocalTablesVal.name() + "需要与topic个数一一对应");
            }
        }
    }
}

Example 2

Source File: JsonSinkClickHouseConnector.java From kafka-connectors with Apache License 2.0

6 votes

vote down

vote up

private void validateTables(ConfigValue sinkTablesVal, ConfigValue sinkLocalTablesVal, String topics) {
    if (sinkTablesVal.value() != null && StringUtils.isNotEmpty(topics)) {
        String sinkTables = (String) sinkTablesVal.value();
        String[] sinkTabArr = sinkTables.split(CONFIG_SEPARATOR);
        String[] topicsArr = topics.split(CONFIG_SEPARATOR);
        if (topicsArr.length != sinkTabArr.length) {
            sinkTablesVal.addErrorMessage(sinkTablesVal.name() + "需要与topic个数一一对应");
        }
        if (sinkLocalTablesVal.value() != null) {
            String[] sinkLocalTabArr = ((String) sinkLocalTablesVal.value()).split(CONFIG_SEPARATOR);
            if (topicsArr.length != sinkLocalTabArr.length) {
                sinkLocalTablesVal.addErrorMessage(sinkLocalTablesVal.name() + "需要与topic个数一一对应");
            }
        }
    }
}

Example 3

Source File: ClickHouseConfigDef.java From kafka-connectors with Apache License 2.0

6 votes

vote down

vote up

/**
 * 所有参数非空校验
 */
public static Map<String, ConfigValue> emptyValidate(Map<String, String> connectorConfigs, ConfigDef configDef) {
    Map<String, ConfigValue> configValues = new HashMap<>(16);
    for (ConfigDef.ConfigKey configKey : configDef.configKeys().values()) {
        ConfigValue configValue = new ConfigValue(configKey.name);
        String value = connectorConfigs.get(configKey.name);
        if (configKey.importance != ConfigDef.Importance.LOW) {
            if (StringUtils.isEmpty(value)) {
                configValue.addErrorMessage(String.format("%s不能为空", configKey.name));
            }
        }
        configValue.value(value);
        configValues.put(configKey.name, configValue);
    }
    return configValues;
}

Example 4

Source File: ClickHouseConfigDef.java From kafka-connectors with Apache License 2.0

6 votes

vote down

vote up

/**
 * 对分布式表是否存在ClickHouse进行校验
 */
public static void clusterTableExist(JdbcDataSource dataSource, ConfigValue sinkTablesVal, String sinkDb) {
    if (sinkTablesVal.errorMessages().isEmpty() && sinkTablesVal.value() != null && dataSource != null) {
        String sinkTables = (String) sinkTablesVal.value();
        try {
            List<String> showTables = dataSource.showTables(sinkDb);
            String[] sinkTabArr = sinkTables.split(CONFIG_SEPARATOR);
            for (String tab : sinkTabArr) {
                if (!showTables.contains(tab)) {
                    sinkTablesVal.addErrorMessage(String.format("%s, 表: %s, 不存在", sinkTablesVal.name(), tab));
                }
            }
        } catch (SQLException e) {
            sinkTablesVal.addErrorMessage(String.format("%s, 查询ClickHouse失败: %s", sinkTablesVal.name(), e.getMessage()));
            logger.error("查询ClickHouse失败,", e);
        }
    }
}

Example 5

Source File: ClickHouseConfigDef.java From kafka-connectors with Apache License 2.0

6 votes

vote down

vote up

/**
 * 对本地表是否存在ClickHouse进行校验, 本地表会在每个连接下进行校验
 */
public static void localTableExist(JdbcDataSource dataSource, ConfigValue sinkLocalTablesVal, String sinkDb) {
    if (sinkLocalTablesVal.errorMessages().isEmpty() && sinkLocalTablesVal.value() != null && dataSource != null) {
        String sinkLocalTables = (String) sinkLocalTablesVal.value();
        String[] sinkLocalTabArr = sinkLocalTables.split(CONFIG_SEPARATOR);
        for (String tab : sinkLocalTabArr) {
            try {
                if (!dataSource.existAllDs(sinkDb, tab)) {
                    sinkLocalTablesVal.addErrorMessage(String.format("%s, 表: %s, 不存在", sinkLocalTablesVal.name(), tab));
                }
            } catch (SQLException e) {
                logger.error("查询ClickHouse失败,", e);
                sinkLocalTablesVal.addErrorMessage(String.format("%s, 查询ClickHouse失败: %s", sinkLocalTablesVal.name(), e.getMessage()));
            }
        }
    }
}

Example 6

Source File: ClickHouseConfigDef.java From kafka-connectors with Apache License 2.0

6 votes

vote down

vote up

/**
 * 校验sink的时间字段个数是否正确；时间字段是否存在对应的表中
 */
public static void validateSinkDateColumns(Map<String, ConfigValue> configValues, JdbcDataSource dataSource, String sinkDb, ConfigValue sinkTablesVal) {
    ConfigValue sinkDateCols = configValues.get(CLICKHOUSE_SINK_DATE_COLUMNS);
    if (sinkTablesVal.errorMessages().isEmpty() && sinkTablesVal.value() != null && dataSource != null) {
        if (sinkDateCols.errorMessages().isEmpty() && sinkDateCols.value() != null) {
            String[] sinkTabArr = ((String) sinkTablesVal.value()).split(CONFIG_SEPARATOR);
            String[] sinkDateColArr = ((String) sinkDateCols.value()).split(CONFIG_SEPARATOR);
            if (sinkDateColArr.length != sinkTabArr.length) {
                sinkDateCols.addErrorMessage(sinkTablesVal.name() + "需要与sink table个数一一对应");
                return;
            }
            for (int i = 0; i < sinkTabArr.length; i++) {
                try {
                    List<String> columns = dataSource.descTable(String.format("`%s`.`%s`", sinkDb, sinkTabArr[i]));
                    if (!columns.contains(sinkDateColArr[i])) {
                        sinkDateCols.addErrorMessage(String.format("%s, 字段: %s, 不存在table: %s中", sinkTablesVal.name(), sinkDateColArr[i], sinkTabArr[i]));
                    }
                } catch (SQLException e) {
                    logger.error("查询ClickHouse失败,", e);
                    sinkDateCols.addErrorMessage(String.format("%s, 查询ClickHouse失败: %s", sinkDateCols.name(), e.getMessage()));
                }
            }
        }
    }
}

Example 7

Source File: ClickHouseConfigDef.java From kafka-connectors with Apache License 2.0

6 votes

vote down

vote up

/**
 * 校验输入的日期格式如果不为空其个数是否与sink table 一致且是否是正确的日期格式化格式
 */
public static void validateDateFormat(Map<String, ConfigValue> configValues, JdbcDataSource dataSource, ConfigValue sinkTablesVal) {
    ConfigValue dateFormat = configValues.get(CLICKHOUSE_SOURCE_DATE_FORMAT);
    if (sinkTablesVal.errorMessages().isEmpty() && sinkTablesVal.value() != null && dataSource != null) {
        if (dateFormat.errorMessages().isEmpty() && dateFormat.value() != null) {
            String[] sinkTabArr = ((String) sinkTablesVal.value()).split(CONFIG_SEPARATOR);
            String[] dateFormatArr = ((String) dateFormat.value()).split(CONFIG_SEPARATOR);
            if (dateFormatArr.length != sinkTabArr.length) {
                dateFormat.addErrorMessage(dateFormat.name() + "需要与sink table个数一一对应");
                return;
            }
            for (String df : dateFormatArr) {
                try {
                    SimpleDateFormat sdf = new SimpleDateFormat(df);
                    sdf.format(new Date());
                } catch (Exception e) {
                    logger.error("校验输入时间格式: ", e);
                    dateFormat.addErrorMessage(String.format("%s, 值: %s, 不是正确的日期格式化格式", dateFormat.name(), df));
                }
            }
        }
    }
}

Example 8

Source File: MySqlSinkClickHouseConnector.java From kafka-connectors with Apache License 2.0

5 votes

vote down

vote up

/**
 * 校验输入的分布式表参数
 * 如果其值不为空, 则判断其个数是否为一个或者其个数是否与topic个数一一对应
 */
private void clusterTableValidate(ConfigValue sinkTablesVal, String topics) {
    if (sinkTablesVal.value() != null) {
        String sinkTables = (String) sinkTablesVal.value();
        String[] sinkTabArr = sinkTables.split(CONFIG_SEPARATOR);
        String[] topicsArr = topics.split(CONFIG_SEPARATOR);
        if (topicsArr.length != sinkTabArr.length) {
            sinkTablesVal.addErrorMessage(sinkTablesVal.name() + "需要与topic个数一一对应");
        }
    }
}

Example 9

Source File: ClickHouseConfigDef.java From kafka-connectors with Apache License 2.0

5 votes

vote down

vote up

/**
 * 对ClickHouse连接进行校验
 */
public static JdbcDataSource clickHouseConnectValidate(Map<String, ConfigValue> configValues) {
    ConfigValue hosts = configValues.get(CLICKHOUSE_HOSTS);
    ConfigValue port = configValues.get(CLICKHOUSE_JDBC_PORT);
    ConfigValue user = configValues.get(CLICKHOUSE_JDBC_USER);
    ConfigValue password = configValues.get(CLICKHOUSE_JDBC_PASSWORD);
    ConfigValue sinkDb = configValues.get(CLICKHOUSE_SINK_DATABASE);
    JdbcDataSource dataSource = null;
    if (hosts.errorMessages().isEmpty() && port.errorMessages().isEmpty() && sinkDb.errorMessages().isEmpty()) {
        JdbcConnectConfig connectConfig = JdbcConnectConfig.initCkConnectConfig((String) hosts.value(), (String) port.value(),
                user == null ? "" : (String) user.value(), password == null ? "" : (String) password.value());
        try {
            dataSource = new JdbcDataSource(connectConfig);
            List<String> databases = dataSource.showDatabases();
            String db = (String) sinkDb.value();
            if (!databases.contains(db)) {
                sinkDb.addErrorMessage(String.format("%s数据库不存在", (String) sinkDb.value()));
            }
            dataSource.close();
            dataSource = null;
        } catch (Exception e) {
            logger.error("校验ClickHouse连接失败: ", e);
            hosts.addErrorMessage("连接ClickHouse失败, " + e.getMessage());
        }
    }
    return dataSource;
}

Example 10

Source File: ClickHouseConfigDef.java From kafka-connectors with Apache License 2.0

5 votes

vote down

vote up

/**
 * 校验数据源中的date字段如果不为空其个数是否与sink table 一致
 */
public static void validateSourceDateColumns(Map<String, ConfigValue> configValues, JdbcDataSource dataSource, ConfigValue sinkTablesVal) {
    ConfigValue sourceDateCols = configValues.get(CLICKHOUSE_SOURCE_DATE_COLUMNS);
    if (sinkTablesVal.errorMessages().isEmpty() && sinkTablesVal.value() != null && dataSource != null) {
        if (sourceDateCols.errorMessages().isEmpty() && sourceDateCols.value() != null) {
            String[] sinkTabArr = ((String) sinkTablesVal.value()).split(CONFIG_SEPARATOR);
            String[] sourceDateColArr = ((String) sourceDateCols.value()).split(CONFIG_SEPARATOR);
            if (sourceDateColArr.length != sinkTabArr.length) {
                sourceDateCols.addErrorMessage(sinkTablesVal.name() + "需要与sink table个数一一对应");
            }
        }
    }
}