Java Code Examples for org.apache.kafka.common.Node#port()

public KafkaTopicPartitionLeader(KafkaTopicPartition topicPartition, Node leader) {
	this.topicPartition = topicPartition;
	if (leader == null) {
		this.leaderId = -1;
		this.leaderHost = null;
		this.leaderPort = -1;
	} else {
		this.leaderId = leader.id();
		this.leaderPort = leader.port();
		this.leaderHost = leader.host();
	}
	int cachedHash = (leader == null) ? 14 : leader.hashCode();
	this.cachedHash = 31 * cachedHash + topicPartition.hashCode();
}
 

public KafkaTopicPartitionLeader(KafkaTopicPartition topicPartition, Node leader) {
	this.topicPartition = topicPartition;
	if (leader == null) {
		this.leaderId = -1;
		this.leaderHost = null;
		this.leaderPort = -1;
	} else {
		this.leaderId = leader.id();
		this.leaderPort = leader.port();
		this.leaderHost = leader.host();
	}
	int cachedHash = (leader == null) ? 14 : leader.hashCode();
	this.cachedHash = 31 * cachedHash + topicPartition.hashCode();
}
 

private void throwIfInvalidConfigMakesClusterUnusable() throws ClusterConfigurationError {
    try {
        Logger.trace("calling kafkaAdminClient.findAllBrokers() ");
        final List<Node> nodes = seqAsJavaList(kafkaAdminClient.findAllBrokers());
        final List<String> advertisedListeners = new ArrayList<>();
        for (Node node : nodes) {
            final String host1 = node.host();
            final int port = node.port();
            final String advertisedListener = String.format("%s:%d", host1, port);
            Logger.debug("Found advertised listener: " + advertisedListener);
            advertisedListeners.add(advertisedListener);

            Logger.trace(String.format("Checking if advertised listener '%s' is reachable", host1));
            if (HostnameUtils.isHostnameReachable(host1, ApplicationConstants.HOSTNAME_REACHABLE_TIMEOUT_MS)) {
                Logger.trace("Yes");
                return;
            }
            Logger.trace("No");
        }
        final String msg = String.format("Cluster config for 'advertised.listeners' is invalid.%n%n" +
                                             "* None of advertised listeners '%s' are reachable from outside world.%n" +
                                             "* Producers/consumers will be unable to use this kafka cluster " +
                                             "(e.g. will not connect properly).%n" +
                                             "* This application (%s) cannot fetch broker configuration", advertisedListeners,
                                         APPLICATION_NAME);
        throw new ClusterConfigurationError(msg);
    } catch (RuntimeException e) {
        Logger.trace(e);
        e.printStackTrace();
        throw e;

    }
}
 

/**
 * This test calls getKafkaBrokers() after the cluster has been properly started. It is expected
 * to return proper connect strings for each of the brokers.
 */
@Test
void testGetKafkaConnectString() throws Exception {
    final int numberOfBrokers = 3;

    try (final KafkaTestCluster kafkaTestCluster = new KafkaTestCluster(numberOfBrokers)) {
        // Start cluster
        kafkaTestCluster.start();

        // Create test Utils
        final KafkaTestUtils kafkaTestUtils = new KafkaTestUtils(kafkaTestCluster);

        // Ask for the connect string
        final String resultStr = kafkaTestCluster.getKafkaConnectString();
        Assertions.assertNotNull(resultStr, "Should have non-null result");

        // Split the result by commas to get individual hosts.
        final Set<String> hosts = new HashSet<>(Arrays.asList(resultStr.split(",")));
        Assertions.assertEquals(numberOfBrokers, hosts.size(), "Should contain 3 entries.");

        // Ask for which nodes exist in the cluster
        final List<Node> nodes = kafkaTestUtils.describeClusterNodes();

        // Sanity test
        Assertions.assertEquals(numberOfBrokers, nodes.size(), "Should have 3 brokers in the cluster");

        // Make sure each node is represented properly.
        for (final Node node: nodes) {
            final String calculatedConnectString = "PLAINTEXT://" + node.host() + ":" + node.port();
            Assertions.assertTrue(hosts.contains(calculatedConnectString), "Should contain " + calculatedConnectString);
        }
    }
}
 

@Override
public boolean startup() {
	Collection<Node> nodes = discoverNodes();
	if (nodes == null || nodes.isEmpty()) {
		return false;
	}

	int poolType = poolCfg.getType();
	String poolName = poolCfg.getName();
	int minCon = poolCfg.getMinCon();
	int maxCon = poolCfg.getMaxCon();

	availableHostList.clear();
	backupHostList.clear();
	for (Node node : nodes) {
		PhysicalNode physicalNode = new PhysicalNode(backendConFactory, 
				poolType, poolName, minCon, maxCon, node.host(), node.port() );
		physicalNode.initConnections();
		physicalNodes.put(node.id(), physicalNode);

		// 防止配置文件未更新的情况
		availableHostList.add(node.host() + ":" + node.port());
		backupHostList.add(node.host() + ":" + node.port());
	}
	
	// 加载 ApiVersion
	loadKafkaVersion();
	
	return true;
}
 

@Override
public int compare(Node a, Node b) {
  int result = a.host().compareTo(b.host());
  if (result == 0) {
    result = a.port() - b.port();
  }
  return result;
}
 

public KafkaTopicPartitionLeader(KafkaTopicPartition topicPartition, Node leader) {
	this.topicPartition = topicPartition;
	if (leader == null) {
		this.leaderId = -1;
		this.leaderHost = null;
		this.leaderPort = -1;
	} else {
		this.leaderId = leader.id();
		this.leaderPort = leader.port();
		this.leaderHost = leader.host();
	}
	int cachedHash = (leader == null) ? 14 : leader.hashCode();
	this.cachedHash = 31 * cachedHash + topicPartition.hashCode();
}
 

/**
 * 
 * 
	zhuamdeMacBook-Pro:logs zhuam$ [2018-05-28 15:26:41,394] INFO [Admin Manager on Broker 0]: 
	Error processing create topic request for topic test01 with arguments (numPartitions=3, replicationFactor=2, replicasAssignments={}, configs={}) (kafka.server.AdminManager)
	org.apache.kafka.common.errors.InvalidReplicationFactorException: Replication factor: 2 larger than available brokers: 1.
 */
private void initializeOfKafka(Map<String, TopicCfg> topicCfgMap) throws Exception, org.apache.kafka.common.errors.TimeoutException {
	
	if (topicCfgMap == null || topicCfgMap.isEmpty()) {
		return;
	}
	
	// Get server address for kafka
	StringBuffer servers = new StringBuffer();
	List<String> nodes = this.getNodes();
	for (int i = 0; i < nodes.size(); i++) {
		String str = nodes.get(i);
		String[] node = str.split(":");
		servers.append(node[0]).append(":").append(node[1]);
		if (i < nodes.size() - 1) {
			servers.append(",");
		}
	}
	
	KafkaAdmin kafkaAdmin = null;
	try {
		// 获取 Kafka 管理对象
		kafkaAdmin = KafkaAdmin.create( servers.toString() );
		
		// 获取kafka中的topic情况
		Map<String, TopicDescription> remoteTopics = kafkaAdmin.getTopicAndDescriptions();
		Collection<Node> clusterNodes = kafkaAdmin.getClusterNodes();
		for (TopicCfg topicCfg : topicCfgMap.values()) {

			String topicName = topicCfg.getName();
			short replicationFactor = topicCfg.getReplicationFactor();
			int partitionNum = topicCfg.getPartitions();

			TopicDescription topicDescription = remoteTopics.get(topicName);
			if (topicDescription != null) {
				int oldPartitionNum = topicDescription.partitions().size();
				if (partitionNum > oldPartitionNum) {
					// add partition
					kafkaAdmin.addPartitionsForTopic(topicName, partitionNum);
					topicDescription = kafkaAdmin.getDescriptionByTopicName(topicName);
				}
				
			} else {
				//verify
				if(clusterNodes == null || replicationFactor > clusterNodes.size()) {
					throw new Exception( "kafka topicName="+ topicName + ", no enough alive physical nodes for replication");
				}
				
				// create topic
				kafkaAdmin.createTopic(topicName, partitionNum, replicationFactor);
				topicDescription = kafkaAdmin.getDescriptionByTopicName(topicName);
			}
			
			// 
			if ( topicDescription == null) {
				throw new Exception( " kafka topicName=" + topicName  + ", description is null.");
			} 

			//
			String name = topicDescription.name();
			boolean internal = topicDescription.isInternal();
			int partitionSize = topicDescription.partitions().size();
			
			//
			BrokerPartition[] newPartitions = new BrokerPartition[ partitionSize ];
			for (int i = 0; i < partitionSize; i++) {
				
				TopicPartitionInfo partitionInfo =  topicDescription.partitions().get(i);
				int partition = partitionInfo.partition();
				
				Node leader = partitionInfo.leader();
				BrokerNode newLeader = new BrokerNode(leader.id(), leader.host(), leader.port());
				
				List<Node> replicas = partitionInfo.replicas();
				BrokerNode[] newReplicas = new BrokerNode[replicas.size()];
				for (int j = 0; j < replicas.size(); j++) {
					newReplicas[j] = new BrokerNode(replicas.get(j).id(), replicas.get(j).host(), replicas.get(j).port());
				}
				
				BrokerPartition newPartition = new BrokerPartition(partition, newLeader, newReplicas);
				newPartitions[i] = newPartition;
			}

			topicCfg.setRunningInfo( new BrokerRunningInfo(name, internal, newPartitions) );
		}
		
	} catch(Throwable e) {
		throw new Exception("kafka pool init err: " + servers.toString(), e);
		
	} finally {
		if (kafkaAdmin != null)
			kafkaAdmin.close();
	}
}