org.apache.kafka.clients.CommonClientConfigs Java Examples

private KafkaFactory.KafkaClientModel get(){
    Properties props = new Properties();
    props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "10.104.156.83:9092");
    props.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer");
    props.put("value.serializer", "org.apache.kafka.common.serialization.ByteArraySerializer");
    props.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, "SASL_PLAINTEXT");
    props.put(SaslConfigs.SASL_MECHANISM, "PLAIN");
    props.put("acks", "all");
    props.put("retries", 0);
    props.put("batch.size", 16384);
    props.put("linger.ms", 1);
    props.put("buffer.memory", 33554432);
    props.put("sasl.jaas.config",
            "org.apache.kafka.common.security.plain.PlainLoginModule required username='kafka' password='kafka';");
    KafkaProducer<String, Byte[]> producer = new KafkaProducer<>(props);
    AdminClient client = AdminClient.create(props);

    KafkaFactory.KafkaClientModel kafkaClientModel = new KafkaFactory.KafkaClientModel(producer, client);
    return kafkaClientModel;
}
 

/**
 * Get a producer connected to the given broker.
 *
 * @param broker The broker to connect to.
 * @return A producer connected to the given broker.
 */
public static KafkaProducer<String, String> producerFor(CCEmbeddedBroker broker) {
  String bootstrap = broker.plaintextAddr();
  if (bootstrap == null) {
    bootstrap = broker.sslAddr();
  }

  Properties props = new Properties();
  props.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, bootstrap);
  props.put(ProducerConfig.ACKS_CONFIG, "all");
  props.put(ProducerConfig.RETRIES_CONFIG, 0);
  props.put(ProducerConfig.BATCH_SIZE_CONFIG, 16384);
  props.put(ProducerConfig.LINGER_MS_CONFIG, 1);
  props.put(ProducerConfig.BUFFER_MEMORY_CONFIG, 1024 * 1024);
  props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer");
  props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer");

  return new KafkaProducer<>(props);
}
 

public KafkaAvroPublisher(String zkUrl, String topic, String statsProducerPropertiesFile) {
  this.destTopic = topic;
  Properties statsProducerProperties = new Properties();
  Map<String, Object> keyValueMap = new HashMap<>();
  try {
    if (statsProducerPropertiesFile != null) {
      statsProducerProperties.load(new FileInputStream(statsProducerPropertiesFile));
      for (String propertyName : statsProducerProperties.stringPropertyNames()) {
        keyValueMap.put(propertyName, statsProducerProperties.get(propertyName));
      }
    }
  } catch (IOException e) {
    LOG.error("Failed to load configuration file {}", statsProducerPropertiesFile, e);
  }
  // set the security protocol based on
  SecurityProtocol securityProtocol = SecurityProtocol.PLAINTEXT;
  if (keyValueMap.containsKey(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG)) {
    String secStr = keyValueMap.get(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG).toString();
    securityProtocol = Enum.valueOf(SecurityProtocol.class, secStr);
  }
  Properties producerProperties = OperatorUtil.createKafkaProducerProperties(zkUrl, securityProtocol);
  for (Map.Entry<String, Object> entry: keyValueMap.entrySet()) {
    producerProperties.put(entry.getKey(), entry.getValue());
  }
  this.kafkaProducer = new KafkaProducer<>(producerProperties);
}
 

public ProducerProxy(ConfigurationManager configurationManager, KafkaMetaManager metaManager,
        KafkaProducerConfig kafkaProducerConfig) {
    ObjectExtension.requireNonNull(configurationManager, "configurationManager");
    ObjectExtension.requireNonNull(metaManager, "metaManager");
    ObjectExtension.requireNonNull(kafkaProducerConfig, "kafkaProducerConfig");

    _clientId = kafkaProducerConfig.getProperties().getProperty(CommonClientConfigs.CLIENT_ID_CONFIG);
    if (StringExtension.isBlank(_clientId))
        _clientId = StringExtension.EMPTY;

    _metaManager = metaManager;
    _kafkaProducerConfig = kafkaProducerConfig;

    _addRemoveLock = new Object();
    _producerHolders = new ConcurrentHashMap<>();

    StringProperties stringProperties = new StringProperties(configurationManager);
    _destroyDelay = stringProperties.getIntProperty("kafka.producer-proxy.producer-destroy-delay", 60 * 1000,
            v -> v < 0 ? null : v);
    _executorService = Executors.newSingleThreadScheduledExecutor(r -> {
        Thread thread = new Thread(r, "kafka.producer-proxy.scheduled-executor");
        thread.setDaemon(true);
        return thread;
    });
}
 

/**
 * If SSL is configured for this cluster, apply the settings.
 * @param clusterConfig Cluster configuration definition to source values from.
 * @param config Config map to apply settings to.
 */
private void applySslSettings(final ClusterConfig clusterConfig, final Map<String, Object> config) {
    // Optionally configure SSL
    if (!clusterConfig.isUseSsl()) {
        return;
    }
    if (clusterConfig.isUseSasl()) {
        config.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, SecurityProtocol.SASL_SSL.name);
    } else {
        config.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, SecurityProtocol.SSL.name);

        // KeyStore and KeyStore password only needed if NOT using SASL
        config.put(SslConfigs.SSL_KEYSTORE_LOCATION_CONFIG, keyStoreRootPath + "/" + clusterConfig.getKeyStoreFile());
        config.put(SslConfigs.SSL_KEYSTORE_PASSWORD_CONFIG, clusterConfig.getKeyStorePassword());
    }
    // Only put Trust properties if one is defined
    if (clusterConfig.getTrustStoreFile() != null) {
        config.put(SslConfigs.SSL_TRUSTSTORE_LOCATION_CONFIG, keyStoreRootPath + "/" + clusterConfig.getTrustStoreFile());
        config.put(SslConfigs.SSL_TRUSTSTORE_PASSWORD_CONFIG, clusterConfig.getTrustStorePassword());
    }
}
 

private void createTopics(int targetPort, Integer numPartitions) throws InterruptedException, java.util.concurrent.ExecutionException {
    List<String> topics = embeddedConfiguration.getTopics();

    if (LOG.isDebugEnabled()) {
        LOG.debug("Creating Kafka Topics in Embedded Kafka: {}", topics);
    }
    if (!topics.isEmpty()) {
        Properties properties = new Properties();
        properties.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, ("127.0.0.1:" + targetPort));
        AdminClient adminClient = AdminClient.create(properties);
        final CreateTopicsResult result = adminClient.createTopics(topics.stream().map(s ->
                new NewTopic(s, numPartitions, (short) 1)).collect(Collectors.toList())
        );
        result.all().get();

        if (LOG.isInfoEnabled()) {
            LOG.info("Created Kafka Topics in Embedded Kafka: {}", topics);
        }
    }
}
 

@Test(timeout = 120000)
public void isKafkaReadyWithSASLAndSSL() throws Exception {
  Properties clientSecurityProps = kafka.getClientSecurityConfig();

  Map<String, String> config = Utils.propsToStringMap(clientSecurityProps);
  config.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, kafka.getBootstrapBroker
      (SecurityProtocol.SASL_SSL));

  // Set password and enabled protocol as the Utils.propsToStringMap just returns toString()
  // representations and these properties don't have a valid representation.
  Password trustStorePassword = (Password) clientSecurityProps.get("ssl.truststore.password");
  config.put("ssl.truststore.password", trustStorePassword.value());
  config.put("ssl.enabled.protocols", "TLSv1.2");

  assertThat(ClusterStatus.isKafkaReady(config, 3, 10000)).isTrue();
}
 

public void rerunTopology(String topologyCode, String ctrlMsg) {
    KafkaProducer<String, byte[]> producer = null;
    try {
        String topic = StringUtils.joinWith("_", topologyCode, "ctrl");
        Properties props = zkService.getProperties(KeeperConstants.KEEPER_CTLMSG_PRODUCER_CONF);
        Properties globalConf = zkService.getProperties(KeeperConstants.GLOBAL_CONF);
        props.setProperty(GLOBAL_CONF_KEY_BOOTSTRAP_SERVERS, globalConf.getProperty(GLOBAL_CONF_KEY_BOOTSTRAP_SERVERS));
        if (StringUtils.equals(SecurityConfProvider.getSecurityConf(zkService), Constants.SECURITY_CONFIG_TRUE_VALUE)) {
            props.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, "SASL_PLAINTEXT");
        }
        producer = new KafkaProducer<>(props);
        producer.send(new ProducerRecord<String, byte[]>(topic, ctrlMsg.getBytes()), new Callback() {
            @Override
            public void onCompletion(RecordMetadata metadata, Exception exception) {
            }
        });
    } catch (Exception e) {
        throw new RuntimeException(e);
    } finally {
        if (producer != null) producer.close();
    }
}
 

@Override
public KafkaConsumer<String, byte[]> obtainKafkaClient(String url) {
    KafkaConsumer<String, byte[]> consumer = null;
    try {
        Properties props = Optional.ofNullable(PropertiesUtils.copy(properties)).orElseGet(() -> {return new Properties();});
        props.setProperty("bootstrap.servers", url);
        props.setProperty("group.id", StringUtils.joinWith("-", props.getProperty("group.id"), suffix));
        props.setProperty("client.id", StringUtils.joinWith("-", props.getProperty("client.id"), suffix));
        if (StringUtils.equals(securityConf, Constants.SECURITY_CONFIG_TRUE_VALUE))
            props.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, "SASL_PLAINTEXT");
        consumer = new KafkaConsumer<>(props);
        logger.info("consumer manager create kafka consumer. url:{}", url);
    } catch (Exception e) {
        logger.error("consumer manager create kafka consumer error. url:{}", url);
        throw new RuntimeException(e);
    }
    return consumer;
}
 

public Properties loadSinkerConf(String confName) throws Exception {
    String path = sinkerRootPath + "/" + confName;
    try {
        Properties properties = zkService.getProperties(path);
        if (confName.equals(SinkerConstants.CONSUMER) || confName.equals(SinkerConstants.PRODUCER)) {
            Properties globalProp = zkService.getProperties(Constants.GLOBAL_PROPERTIES_ROOT);
            properties.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, globalProp.getProperty(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG));
            if (isSecurityConf()) {
                properties.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, "SASL_PLAINTEXT");
            }
        }
        logger.info("load zk properties -- {} : {}", path, properties);
        return properties;
    } catch (Exception e) {
        logger.error("load sinker zk node [{}] properties error.", confName, e);
        throw e;
    }
}
 

@Override
public void record(TaskExceptionProbeIndex index) {
    String bootMode = WorkerConfig.current().getString(WorkerConfig.WORKER_BOOT_MODE_CONFIG);
    if (index.isSendDirectly() && bootMode.equals("distributed")) {
        sendDirectly(index);
    } else {
        cache.put(index.getTaskId(), index.getExceptionInfo());
    }
    try {
        TaskExceptionInfo taskExceptionInfo = new TaskExceptionInfo();
        String clientId = WorkerConfig.current().getString(CommonClientConfigs.CLIENT_ID_CONFIG);//???
        taskExceptionInfo.setWorkerId(Long.valueOf(clientId));
        taskExceptionInfo.setTaskId(index.getTaskId());
        taskExceptionInfo.setExceptionDetail(index.getExceptionInfo());
        taskExceptionService.insert(taskExceptionInfo);
    } catch (Exception e) {
        logger.info("Record exception detail failed.", e);
    }
}
 

@Test(timeout = 120000)
public void isKafkaReadyWithSASLAndSSLUsingZK() throws Exception {
  Properties clientSecurityProps = kafka.getClientSecurityConfig();

  boolean zkReady = ClusterStatus.isZookeeperReady(this.kafka.getZookeeperConnectString(), 30000);
  if (!zkReady) {
    throw new RuntimeException(
        "Could not reach zookeeper " + this.kafka.getZookeeperConnectString());
  }
  Map<String, String> endpoints = ClusterStatus.getKafkaEndpointFromZookeeper(
      this.kafka.getZookeeperConnectString(),
      30000
  );

  String bootstrap_broker = endpoints.get("SASL_SSL");
  Map<String, String> config = Utils.propsToStringMap(clientSecurityProps);
  config.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, bootstrap_broker);

  // Set password and enabled protocol as the Utils.propsToStringMap just returns toString()
  // representations and these properties don't have a valid representation.
  Password trustStorePassword = (Password) clientSecurityProps.get("ssl.truststore.password");
  config.put("ssl.truststore.password", trustStorePassword.value());
  config.put("ssl.enabled.protocols", "TLSv1.2");

  assertThat(ClusterStatus.isKafkaReady(config, 3, 10000)).isTrue();
}
 

public Map<TopicPartition, Long> getKafkaLogSize(String topic, Set<Integer> partitionids) {
	Properties props = new Properties();
	props.put(ConsumerConfig.GROUP_ID_CONFIG, Kafka.KAFKA_EAGLE_SYSTEM_GROUP);
	props.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, "127.0.0.1:9092");
	props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getCanonicalName());
	props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getCanonicalName());
	KafkaConsumer<String, String> consumer = new KafkaConsumer<>(props);
	Set<TopicPartition> tps = new HashSet<>();
	Map<Integer, Long> partitionOffset = new HashMap<Integer, Long>();
	for (int partitionid : partitionids) {
		TopicPartition tp = new TopicPartition(topic, partitionid);
		long offset = consumer.position(tp);
		partitionOffset.put(partitionid, offset);
	}

	System.out.println(partitionOffset.toString());

	if (consumer != null) {
		consumer.close();
	}
	return null;
}
 

public Properties loadKafkaConsumerConf() {
    Properties consumerProps;
    String path = Constants.TOPOLOGY_ROOT + "/" + topologyId + "-" + LOG_PROCESSOR + "/consumer.properties";
    try {
        consumerProps = zkService.getProperties(path);
    } catch (Exception e) {
        e.printStackTrace();
        logger.error("load consumer.properties error.", e);
        throw new RuntimeException("load consumer.properties error", e);
    }

    Properties props = loadSecurityConf();
    if (props != null) {
        if (StringUtils.equals(props.getProperty("AuthenticationAndAuthorization"), Constants.SECURITY_CONFIG_TRUE_VALUE)) {
            consumerProps.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, "SASL_PLAINTEXT");
            logger.info("consumer security_protocol is enabled!  security_protocol_config is: SASL_PLAINTEXT");
        } else if (StringUtils.equals(props.getProperty("AuthenticationAndAuthorization"), "none")) {
            logger.info("consumer security_protocol is disabled!");
        }
    }

    return consumerProps;
}
 

/**
 * Send mock message to kafka topic .
 */
public boolean mockMessage(String clusterAlias, String topic, String message) {
	Properties props = new Properties();
	props.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, getKafkaBrokerServer(clusterAlias));
	props.put(Kafka.KEY_SERIALIZER, StringSerializer.class.getCanonicalName());
	props.put(Kafka.VALUE_SERIALIZER, StringSerializer.class.getCanonicalName());
	props.put(Kafka.PARTITION_CLASS, KafkaPartitioner.class.getName());

	if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.sasl.enable")) {
		sasl(props, clusterAlias);
	}
	if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.ssl.enable")) {
		ssl(props, clusterAlias);
	}
	Producer<String, String> producer = new KafkaProducer<>(props);
	producer.send(new ProducerRecord<String, String>(topic, new Date().getTime() + "", message));
	producer.close();

	return true;
}
 

private void addSecurityConf(Properties properties) throws Exception {
    if (isSecurity == null) {
        synchronized (this.getClass()) {
            if (isSecurity == null) {
                String path = Constants.COMMON_ROOT + "/" + Constants.GLOBAL_SECURITY_CONF;
                if (zookeeper.isExists(path)) {
                    Properties pro = zookeeper.getProperties(path);
                    if (StringUtils.equals(pro.getProperty("AuthenticationAndAuthorization"), Constants.SECURITY_CONFIG_TRUE_VALUE)) {
                        isSecurity = true;
                    } else {
                        isSecurity = false;
                    }
                } else {
                    isSecurity = false;
                }
            }
        }
    }
    if (isSecurity) {
        properties.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, "SASL_PLAINTEXT");
    }
}
 

@Override
public Properties overridingProps() {
    Properties props = new Properties();
    int port = CCKafkaTestUtils.findLocalPort();
    props.setProperty(CommonClientConfigs.METRIC_REPORTER_CLASSES_CONFIG, CruiseControlMetricsReporter.class.getName());
    props.setProperty(KafkaConfig.ListenersProp(), "PLAINTEXT://127.0.0.1:" + port);
    props.setProperty(CruiseControlMetricsReporterConfig.config(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG),
            "127.0.0.1:" + port);
    props.setProperty(CruiseControlMetricsReporterConfig.CRUISE_CONTROL_METRICS_REPORTER_INTERVAL_MS_CONFIG, "100");
    props.setProperty(CruiseControlMetricsReporterConfig.CRUISE_CONTROL_METRICS_TOPIC_CONFIG, TOPIC);
    // configure metrics topic auto-creation by the metrics reporter
    props.setProperty(CruiseControlMetricsReporterConfig.CRUISE_CONTROL_METRICS_TOPIC_AUTO_CREATE_CONFIG, "true");
    props.setProperty(CruiseControlMetricsReporterConfig.CRUISE_CONTROL_METRICS_TOPIC_AUTO_CREATE_TIMEOUT_MS_CONFIG, "5000");
    props.setProperty(CruiseControlMetricsReporterConfig.CRUISE_CONTROL_METRICS_TOPIC_AUTO_CREATE_RETRIES_CONFIG, "1");
    props.setProperty(CruiseControlMetricsReporterConfig.CRUISE_CONTROL_METRICS_TOPIC_NUM_PARTITIONS_CONFIG, "1");
    props.setProperty(CruiseControlMetricsReporterConfig.CRUISE_CONTROL_METRICS_TOPIC_REPLICATION_FACTOR_CONFIG, "1");
    // disable topic auto-creation to leave the metrics reporter to create the metrics topic
    props.setProperty(KafkaConfig.AutoCreateTopicsEnableProp(), "false");
    props.setProperty(KafkaConfig.LogFlushIntervalMessagesProp(), "1");
    props.setProperty(KafkaConfig.OffsetsTopicReplicationFactorProp(), "1");
    props.setProperty(KafkaConfig.DefaultReplicationFactorProp(), "2");
    props.setProperty(KafkaConfig.NumPartitionsProp(), "2");
    return props;
}
 

@Override
public Properties overridingProps() {
  Properties props = new Properties();
  int port = CCKafkaTestUtils.findLocalPort();
  // We need to convert all the properties to the Cruise Control properties.
  setSecurityConfigs(props, "producer");
  for (String configName : ProducerConfig.configNames()) {
    Object value = props.get(configName);
    if (value != null) {
      props.remove(configName);
      props.put(appendPrefix(configName), value);
    }
  }
  props.setProperty(CommonClientConfigs.METRIC_REPORTER_CLASSES_CONFIG, CruiseControlMetricsReporter.class.getName());
  props.setProperty(KafkaConfig.ListenersProp(), "SSL://127.0.0.1:" + port);
  props.setProperty(CruiseControlMetricsReporterConfig.config(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG), "127.0.0.1:" + port);
  props.setProperty(CruiseControlMetricsReporterConfig.config(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG), SecurityProtocol.SSL.name);
  props.setProperty(CRUISE_CONTROL_METRICS_REPORTER_INTERVAL_MS_CONFIG, "100");
  props.setProperty(CRUISE_CONTROL_METRICS_TOPIC_CONFIG, TOPIC);
  props.setProperty(KafkaConfig.LogFlushIntervalMessagesProp(), "1");
  props.setProperty(KafkaConfig.OffsetsTopicReplicationFactorProp(), "1");
  props.setProperty(KafkaConfig.DefaultReplicationFactorProp(), "2");
  return props;
}
 

protected void setSecurityConfigs(Properties clientProps, String certAlias) {
  SecurityProtocol protocol = securityProtocol();
  if (protocol == SecurityProtocol.SSL) {
    File trustStoreFile = trustStoreFile();
    if (trustStoreFile == null) {
      throw new AssertionError("ssl set but no trust store provided");
    }
    clientProps.setProperty(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, protocol.name);
    clientProps.setProperty(KafkaConfig.SslEndpointIdentificationAlgorithmProp(), "");
    try {
      clientProps.putAll(TestSslUtils.createSslConfig(true, true, Mode.CLIENT, trustStoreFile, certAlias));
    } catch (Exception e) {
      throw new IllegalStateException(e);
    }
  }
}
 

/**
 * Get a consumer connected to the given broker.
 *
 * @param broker The broker to connect to.
 * @return A consumer connected to the given broker.
 */
public static KafkaConsumer<String, String> consumerFor(CCEmbeddedBroker broker) {
  String bootstrap = broker.plaintextAddr();
  if (bootstrap == null) {
    bootstrap = broker.sslAddr();
  }

  Properties props = new Properties();
  props.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, bootstrap);
  props.put(ConsumerConfig.GROUP_ID_CONFIG, "test");
  props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
  props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "false");
  props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringDeserializer");
  props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringDeserializer");

  return new KafkaConsumer<>(props);
}
 

@Override
public ExternalCatalogTable getTable(String tableName) throws TableNotExistException {
  Map<String, String> sourceTableProp = new HashMap<>();
  sourceTableProp.put(KAFKA_CONFIG_PREFIX + "." + ConsumerConfig.GROUP_ID_CONFIG, tableName);
  sourceTableProp.put(KAFKA_CONFIG_PREFIX + "." + CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, broker);
  sourceTableProp.put(KAFKA_CONFIG_PREFIX + "." + ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
  sourceTableProp.put(TOPIC_NAME_KEY, tableName);
  return getKafkaExternalCatalogTable(sourceTableProp);
}
 

/**
 * Get kafka 0.10.x topic history logsize.
 */
public long getKafkaLogSize(String clusterAlias, String topic, int partitionid) {
	long histyLogSize = 0L;
	Properties props = new Properties();
	props.put(ConsumerConfig.GROUP_ID_CONFIG, Kafka.KAFKA_EAGLE_SYSTEM_GROUP);
	props.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, getKafkaBrokerServer(clusterAlias));
	props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getCanonicalName());
	props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getCanonicalName());
	if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.sasl.enable")) {
		sasl(props, clusterAlias);
	}
	if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.ssl.enable")) {
		ssl(props, clusterAlias);
	}
	KafkaConsumer<String, String> consumer = new KafkaConsumer<>(props);
	TopicPartition tp = new TopicPartition(topic, partitionid);
	consumer.assign(Collections.singleton(tp));
	java.util.Map<TopicPartition, Long> logsize = consumer.endOffsets(Collections.singleton(tp));
	try {
		histyLogSize = logsize.get(tp).longValue();
	} catch (Exception e) {
		LOG.error("Get history topic logsize has error, msg is " + e.getMessage());
		e.printStackTrace();
	} finally {
		if (consumer != null) {
			consumer.close();
		}
	}
	return histyLogSize;
}
 

private ProducerSettings<String, String> addAuthenticatedSecurityProtocol(
        final ProducerSettings<String, String> settings) {

    if (isConnectionSecure()) {
        return settings.withProperty(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, "SASL_SSL");
    }
    return settings.withProperty(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, "SASL_PLAINTEXT");
}
 

/**
 * Get kafka 0.10.x, 1.x, 2.x consumer groups.
 */
public int getKafkaConsumerGroups(String clusterAlias) {
	Properties prop = new Properties();
	int counter = 0;
	prop.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, parseBrokerServer(clusterAlias));

	if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.sasl.enable")) {
		sasl(prop, clusterAlias);
	}
	if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.ssl.enable")) {
		ssl(prop, clusterAlias);
	}

	AdminClient adminClient = null;
	try {
		adminClient = AdminClient.create(prop);
		ListConsumerGroupsResult consumerGroups = adminClient.listConsumerGroups();
		java.util.Iterator<ConsumerGroupListing> groups = consumerGroups.all().get().iterator();
		while (groups.hasNext()) {
			String groupId = groups.next().groupId();
			if (!groupId.contains("kafka.eagle")) {
				counter++;
			}
		}
	} catch (Exception e) {
		LOG.info("Get kafka consumer group has error, msg is " + e.getMessage());
		e.printStackTrace();
	} finally {
		adminClient.close();
	}
	return counter;
}
 

/**
 * Get kafka 0.10.x, 1.x, 2.x offset from topic.
 */
public String getKafkaOffset(String clusterAlias) {
	Properties prop = new Properties();
	prop.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, parseBrokerServer(clusterAlias));

	if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.sasl.enable")) {
		sasl(prop, clusterAlias);
	}
	if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.ssl.enable")) {
		ssl(prop, clusterAlias);
	}
	JSONArray targets = new JSONArray();
	AdminClient adminClient = null;
	try {
		adminClient = AdminClient.create(prop);
		ListConsumerGroupsResult consumerGroups = adminClient.listConsumerGroups();
		java.util.Iterator<ConsumerGroupListing> groups = consumerGroups.all().get().iterator();
		while (groups.hasNext()) {
			String groupId = groups.next().groupId();
			if (!groupId.contains("kafka.eagle")) {
				ListConsumerGroupOffsetsResult offsets = adminClient.listConsumerGroupOffsets(groupId);
				for (Entry<TopicPartition, OffsetAndMetadata> entry : offsets.partitionsToOffsetAndMetadata().get().entrySet()) {
					JSONObject object = new JSONObject();
					object.put("group", groupId);
					object.put("topic", entry.getKey().topic());
					object.put("partition", entry.getKey().partition());
					object.put("offset", entry.getValue().offset());
					object.put("timestamp", CalendarUtils.getDate());
					targets.add(object);
				}
			}
		}
	} catch (Exception e) {
		LOG.error("Get consumer offset has error, msg is " + e.getMessage());
		e.printStackTrace();
	} finally {
		adminClient.close();
	}
	return targets.toJSONString();
}
 

public void loadKafkaProducerConfig() {
    //TODO
    String path = extractorRoot + extractorName + "/producer.properties";
    try {
        Properties kafkaProducerConfigs = zkService.getProperties(path);
        if (StringUtils.equals(SecurityConfProvider.getSecurityConf(zkService), Constants.SECURITY_CONFIG_TRUE_VALUE)) {
            kafkaProducerConfigs.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, "SASL_PLAINTEXT");
        }
        ExtractorConfigContainer.getInstances().setKafkaProducerConfig(kafkaProducerConfigs);
        logger.info("kafka producer configs:{}", kafkaProducerConfigs);
    } catch (Exception e) {
        throw new RuntimeException("load extractor config: " + path + "配置信息出错.", e);
    }
}
 

/**
 * Set topic ssl.
 */
public void ssl(Properties props, String clusterAlias) {
	// configure the following three settings for SSL Encryption
	props.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, SystemConfigUtils.getProperty(clusterAlias + ".kafka.eagle.ssl.protocol"));
	props.put(SslConfigs.SSL_TRUSTSTORE_LOCATION_CONFIG, SystemConfigUtils.getProperty(clusterAlias + ".kafka.eagle.ssl.truststore.location"));
	props.put(SslConfigs.SSL_TRUSTSTORE_PASSWORD_CONFIG, SystemConfigUtils.getProperty(clusterAlias + ".kafka.eagle.ssl.truststore.password"));

	// configure the following three settings for SSL Authentication
	props.put(SslConfigs.SSL_KEYSTORE_LOCATION_CONFIG, SystemConfigUtils.getProperty(clusterAlias + ".kafka.eagle.ssl.keystore.location"));
	props.put(SslConfigs.SSL_KEYSTORE_PASSWORD_CONFIG, SystemConfigUtils.getProperty(clusterAlias + ".kafka.eagle.ssl.keystore.password"));
	props.put(SslConfigs.SSL_KEY_PASSWORD_CONFIG, SystemConfigUtils.getProperty(clusterAlias + ".kafka.eagle.ssl.key.password"));

}
 

public Properties getProducerProps() throws Exception {
    String path = topologyRoot + "/" + Constants.DISPATCHER_PRODUCER_PROPERTIES;
    Properties props = zkService.getProperties(path);
    String clientID = topologyID + "-producer";
    props.put("client.id", clientID);
    if (StringUtils.equals(SecurityConfProvider.getSecurityConf(zkService), "kerberos_kafkaACL")) {
        props.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, "SASL_PLAINTEXT");
    }
    return props;
}
 

/**
 * Set topic sasl.
 */
public void sasl(Properties props, String clusterAlias) {
	// configure the following four settings for SSL Encryption
	props.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, SystemConfigUtils.getProperty(clusterAlias + ".kafka.eagle.sasl.protocol"));
	if (!"".equals(SystemConfigUtils.getProperty(clusterAlias + ".kafka.eagle.sasl.client.id"))) {
		props.put(CommonClientConfigs.CLIENT_ID_CONFIG, SystemConfigUtils.getProperty(clusterAlias + ".kafka.eagle.sasl.client.id"));
	}
	props.put(SaslConfigs.SASL_MECHANISM, SystemConfigUtils.getProperty(clusterAlias + ".kafka.eagle.sasl.mechanism"));
	props.put(SaslConfigs.SASL_JAAS_CONFIG, SystemConfigUtils.getProperty(clusterAlias + ".kafka.eagle.sasl.jaas.config"));
}
 

public Properties getConsumerProps() throws Exception {
    String path = topologyRoot + "/" + Constants.DISPATCHER_CUNSUMER_PROPERTIES;
    Properties props = zkService.getProperties(path);
    String id = topologyID + "-consumer";
    props.setProperty("group.id", id);
    props.setProperty("client.id", id);
    if (StringUtils.equals(SecurityConfProvider.getSecurityConf(zkService), "kerberos_kafkaACL")) {
        props.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, "SASL_PLAINTEXT");
    }
    logger.info(String.format("Spout set: group.id and client.id=%s", id));
    return props;
}