Java Code Examples for kafka.javaapi.PartitionMetadata#leader()
The following examples show how to use
kafka.javaapi.PartitionMetadata#leader() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
| Source File: KafkaLowLevelConsumer09.java From datacollector with Apache License 2.0 | 6 votes |
|
private HostAndPort findNewLeader(HostAndPort oldLeader, String topic, int partition) throws StageException {
//try 3 times to find a new leader
for (int i = 0; i < 3; i++) {
boolean sleep;
PartitionMetadata metadata = getPartitionMetadata(replicaBrokers, topic, partition);
if (metadata == null || metadata.leader() == null) {
sleep = true;
} else if (oldLeader.getHostText().equalsIgnoreCase(metadata.leader().host()) && i == 0) {
//leader has not yet changed, give zookeeper sometime
sleep = true;
} else {
return HostAndPort.fromParts(metadata.leader().host(), metadata.leader().port());
}
if (sleep) {
ThreadUtil.sleep(ONE_SECOND);
}
}
LOG.error(KafkaErrors.KAFKA_21.getMessage());
throw new StageException(KafkaErrors.KAFKA_21);
}
Example 2
| Source File: AckKafkaWriterTestRunner.java From Scribengin with GNU Affero General Public License v3.0 | 6 votes |
|
public void run() {
try {
while (!exit) {
KafkaTool kafkaTool = new KafkaTool(topic, cluster.getZKConnect());
kafkaTool.connect();
TopicMetadata topicMeta = kafkaTool.findTopicMetadata(topic);
PartitionMetadata partitionMeta = findPartition(topicMeta, partition);
Broker partitionLeader = partitionMeta.leader();
Server kafkaServer = cluster.findKafkaServerByPort(partitionLeader.port());
System.out.println("Shutdown kafka server " + kafkaServer.getPort());
kafkaServer.shutdown();
failureCount++;
Thread.sleep(sleepBeforeRestart);
kafkaServer.start();
kafkaTool.close();
Thread.sleep(10000); //wait to make sure that the kafka server start
}
} catch (Exception e) {
e.printStackTrace();
}
synchronized (this) {
notify();
}
}
Example 3
| Source File: ScribeConsumer.java From Scribengin with GNU Affero General Public License v3.0 | 6 votes |
|
private HostPort findNewLeader(String oldHost, int oldPort) throws LostLeadershipException {
for (int i = 0; i < 3; i++) {
boolean goToSleep = false;
PartitionMetadata metadata = findLeader(replicaBrokers, topic, partition);
if (metadata == null) {
goToSleep = true;
} else if (metadata.leader() == null) {
goToSleep = true;
} else if (oldHost.equalsIgnoreCase(metadata.leader().host()) &&
oldPort == metadata.leader().port()) {
// first time through if the leader hasn't changed give ZooKeeper a second to recover
// second time, assume the broker did recover before failover, or it was a non-Broker issue
goToSleep = true;
} else {
return new HostPort(metadata.leader().host(), metadata.leader().port());
}
if (goToSleep) {
try {
Thread.sleep(1000);
} catch (InterruptedException ie) {
}
}
}
// Can't recover from a leadership disappearance.
throw new LostLeadershipException();
}
Example 4
| Source File: Kafka08ConsumerClient.java From incubator-gobblin with Apache License 2.0 | 6 votes |
|
private List<KafkaPartition> getPartitionsForTopic(TopicMetadata topicMetadata) {
List<KafkaPartition> partitions = Lists.newArrayList();
for (PartitionMetadata partitionMetadata : topicMetadata.partitionsMetadata()) {
if (null == partitionMetadata) {
log.error("Ignoring topic with null partition metadata " + topicMetadata.topic());
return Collections.emptyList();
}
if (null == partitionMetadata.leader()) {
log.error("Ignoring topic with null partition leader " + topicMetadata.topic() + " metatada="
+ partitionMetadata);
return Collections.emptyList();
}
partitions.add(new KafkaPartition.Builder().withId(partitionMetadata.partitionId())
.withTopicName(topicMetadata.topic()).withLeaderId(partitionMetadata.leader().id())
.withLeaderHostAndPort(partitionMetadata.leader().host(), partitionMetadata.leader().port()).build());
}
return partitions;
}
Example 5
| Source File: KafkaWrapper.java From incubator-gobblin with Apache License 2.0 | 6 votes |
|
private List<KafkaPartition> getPartitionsForTopic(TopicMetadata topicMetadata) {
List<KafkaPartition> partitions = Lists.newArrayList();
for (PartitionMetadata partitionMetadata : topicMetadata.partitionsMetadata()) {
if (null == partitionMetadata) {
LOG.error("Ignoring topic with null partition metadata " + topicMetadata.topic());
return Collections.emptyList();
}
if (null == partitionMetadata.leader()) {
LOG.error(
"Ignoring topic with null partition leader " + topicMetadata.topic() + " metatada=" + partitionMetadata);
return Collections.emptyList();
}
partitions.add(new KafkaPartition.Builder().withId(partitionMetadata.partitionId())
.withTopicName(topicMetadata.topic()).withLeaderId(partitionMetadata.leader().id())
.withLeaderHostAndPort(partitionMetadata.leader().host(), partitionMetadata.leader().port()).build());
}
return partitions;
}
Example 6
| Source File: NativeSimpleConsumer.java From spring-kafka-demo with Apache License 2.0 | 6 votes |
|
private String findNewLeader(String a_oldLeader, String a_topic, int a_partition, int a_port) throws Exception {
for (int i = 0; i < 3; i++) {
boolean goToSleep = false;
PartitionMetadata metadata = findLeader(m_replicaBrokers, a_port, a_topic, a_partition);
if (metadata == null) {
goToSleep = true;
} else if (metadata.leader() == null) {
goToSleep = true;
} else if (a_oldLeader.equalsIgnoreCase(metadata.leader().host()) && i == 0) {
// first time through if the leader hasn't changed give ZooKeeper a second to recover
// second time, assume the broker did recover before failover, or it was a non-Broker issue
//
goToSleep = true;
} else {
return metadata.leader().host();
}
if (goToSleep) {
try {
Thread.sleep(1000);
} catch (InterruptedException ie) {
}
}
}
System.out.println("Unable to find new leader after Broker failure. Exiting");
throw new Exception("Unable to find new leader after Broker failure. Exiting");
}
Example 7
| Source File: KafkaSimpleConsumer.java From Pistachio with Apache License 2.0 | 6 votes |
|
private Broker findNewLeader(Broker oldLeader) throws InterruptedException {
long retryCnt = 0;
while (true) {
PartitionMetadata metadata = findLeader();
logger.debug("findNewLeader - meta leader {}, previous leader {}", metadata, oldLeader);
if (metadata != null && metadata.leader() != null && (oldLeader == null ||
(!(oldLeader.host().equalsIgnoreCase(metadata.leader().host()) &&
(oldLeader.port() == metadata.leader().port())) || retryCnt != 0))) {
// first time through if the leader hasn't changed give ZooKeeper a second to recover
// second time, assume the broker did recover before failover, or it was a non-Broker issue
logger.info("findNewLeader - using new leader {} from meta data, previous leader {}", metadata.leader(), oldLeader);
return metadata.leader();
}
//TODO: backoff retry
Thread.sleep(1000L);
retryCnt ++;
// if could not find the leader for current replicaBrokers, let's try to find one via allBrokers
if (retryCnt >= 3 && (retryCnt - 3) % 5 == 0) {
logger.warn("can nof find leader for {} - {} after {} retries", topic, partitionId, retryCnt);
replicaBrokers.clear();
replicaBrokers.addAll(allBrokers);
}
}
}
Example 8
| Source File: SimpleKafkaMessageListener.java From micro-integrator with Apache License 2.0 | 6 votes |
|
public boolean run() throws Exception {
if (init) {
return init;
}
// find the meta data about the topic and partition we are interested in
PartitionMetadata metadata = findLeader(seedBrokers, port, topic, partition);
if (metadata == null) {
throw new SynapseException("Can't find metadata for Topic and Partition. Exiting");
}
if (metadata.leader() == null) {
throw new SynapseException("Can't find Leader for Topic and Partition. Exiting");
}
this.leadBroker = metadata.leader().host();
this.clientName = "Client_" + topic + "_" + partition;
this.consumer = new SimpleConsumer(leadBroker, port, KAFKAConstants.BUFFER_SIZE, KAFKAConstants.SO_TIMEOUT,
clientName);
this.readOffset = getLastOffset(consumer, topic, partition, kafka.api.OffsetRequest.EarliestTime(), clientName);
init = true;
return init;
}
Example 9
| Source File: KafkaLowLevelConsumer08.java From datacollector with Apache License 2.0 | 6 votes |
|
private HostAndPort findNewLeader(HostAndPort oldLeader, String topic, int partition) throws StageException {
//try 3 times to find a new leader
for (int i = 0; i < 3; i++) {
boolean sleep;
PartitionMetadata metadata = getPartitionMetadata(replicaBrokers, topic, partition);
if (metadata == null || metadata.leader() == null) {
sleep = true;
} else if (oldLeader.getHostText().equalsIgnoreCase(metadata.leader().host()) && i == 0) {
//leader has not yet changed, give zookeeper sometime
sleep = true;
} else {
return HostAndPort.fromParts(metadata.leader().host(), metadata.leader().port());
}
if (sleep) {
ThreadUtil.sleep(ONE_SECOND);
}
}
LOG.error(KafkaErrors.KAFKA_21.getMessage());
throw new StageException(KafkaErrors.KAFKA_21);
}
Example 10
| Source File: KafkaTestConsumer.java From hadoop-mini-clusters with Apache License 2.0 | 6 votes |
|
private String findNewLeader(String a_oldLeader, String a_topic, int a_partition, int a_port) throws Exception {
for (int i = 0; i < 3; i++) {
boolean goToSleep = false;
PartitionMetadata metadata = findLeader(m_replicaBrokers, a_port, a_topic, a_partition);
if (metadata == null) {
goToSleep = true;
} else if (metadata.leader() == null) {
goToSleep = true;
} else if (a_oldLeader.equalsIgnoreCase(metadata.leader().host()) && i == 0) {
// first time through if the leader hasn't changed give ZooKeeper a second to recover
// second time, assume the broker did recover before failover, or it was a non-Broker issue
//
goToSleep = true;
} else {
return metadata.leader().host();
}
if (goToSleep) {
try {
Thread.sleep(1000);
} catch (InterruptedException ie) {
}
}
}
LOG.info("Unable to find new leader after Broker failure. Exiting");
throw new Exception("Unable to find new leader after Broker failure. Exiting");
}
Example 11
| Source File: KafkaLatestOffsetFetcher.java From eagle with Apache License 2.0 | 6 votes |
|
public Map<Integer, Long> fetch(String topic, int partitionCount) {
Map<Integer, PartitionMetadata> metadatas = fetchPartitionMetadata(brokerList, port, topic, partitionCount);
Map<Integer, Long> ret = new HashMap<>();
for (int partition = 0; partition < partitionCount; partition++) {
PartitionMetadata metadata = metadatas.get(partition);
if (metadata == null || metadata.leader() == null) {
ret.put(partition, -1L);
//throw new RuntimeException("Can't find Leader for Topic and Partition. Exiting");
}
String leadBroker = metadata.leader().host();
String clientName = "Client_" + topic + "_" + partition;
SimpleConsumer consumer = new SimpleConsumer(leadBroker, port, 100000, 64 * 1024, clientName);
long latestOffset = getLatestOffset(consumer, topic, partition, clientName);
if (consumer != null) {
consumer.close();
}
ret.put(partition, latestOffset);
}
return ret;
}
Example 12
| Source File: SimpleKafkaMessageListener.java From micro-integrator with Apache License 2.0 | 6 votes |
|
private String findNewLeader(String oldLeader, String topic, int partition, int port) throws Exception {
for (int i = 0; i < 3; i++) {
boolean goToSleep = false;
PartitionMetadata metadata = findLeader(replicaBrokers, port, topic, partition);
if (metadata == null) {
goToSleep = true;
} else if (metadata.leader() == null) {
goToSleep = true;
} else if (oldLeader.equalsIgnoreCase(metadata.leader().host()) && i == 0) {
goToSleep = true;
} else {
return metadata.leader().host();
}
if (goToSleep) {
try {
Thread.sleep(1000);
} catch (InterruptedException ie) {
}
}
}
throw new SynapseException("Unable to find new leader after Broker failure. Exiting");
}
Example 13
| Source File: KafkaLowLevelConsumer09.java From datacollector with Apache License 2.0 | 5 votes |
|
@Override
public void init() throws StageException {
List<HostAndPort> brokers = new ArrayList<>();
brokers.add(broker);
PartitionMetadata metadata = getPartitionMetadata(brokers, topic, partition);
if (metadata == null) {
LOG.error(KafkaErrors.KAFKA_23.getMessage(), topic, partition);
throw new StageException(KafkaErrors.KAFKA_23, topic, partition);
}
if (metadata.leader() == null) {
LOG.error(KafkaErrors.KAFKA_24.getMessage(), topic, partition);
throw new StageException(KafkaErrors.KAFKA_24, topic, partition);
}
leader = HostAndPort.fromParts(metadata.leader().host(), metadata.leader().port());
//recreate consumer instance with the leader information for that topic
LOG.info(
"Creating SimpleConsumer using the following configuration: host {}, port {}, max wait time {}, max " +
"fetch size {}, client columnName {}",
leader.getHostText(),
leader.getPort(),
maxWaitTime,
maxFetchSize,
clientName
);
consumer = new SimpleConsumer(
leader.getHostText(),
leader.getPort(),
maxWaitTime,
maxFetchSize,
clientName
);
}
Example 14
| Source File: KafkaLowLevelConsumer08.java From datacollector with Apache License 2.0 | 5 votes |
|
@Override
public void init() throws StageException {
List<HostAndPort> brokers = new ArrayList<>();
brokers.add(broker);
PartitionMetadata metadata = getPartitionMetadata(brokers, topic, partition);
if (metadata == null) {
LOG.error(KafkaErrors.KAFKA_23.getMessage(), topic, partition);
throw new StageException(KafkaErrors.KAFKA_23, topic, partition);
}
if (metadata.leader() == null) {
LOG.error(KafkaErrors.KAFKA_24.getMessage(), topic, partition);
throw new StageException(KafkaErrors.KAFKA_24, topic, partition);
}
leader = HostAndPort.fromParts(metadata.leader().host(), metadata.leader().port());
//recreate consumer instance with the leader information for that topic
LOG.info(
"Creating SimpleConsumer using the following configuration: host {}, port {}, max wait time {}, max " +
"fetch size {}, client columnName {}",
leader.getHostText(),
leader.getPort(),
maxWaitTime,
maxFetchSize,
clientName
);
consumer = new SimpleConsumer(
leader.getHostText(),
leader.getPort(),
maxWaitTime,
maxFetchSize,
clientName
);
}
Example 15
| Source File: ScribeConsumer.java From Scribengin with GNU Affero General Public License v3.0 | 5 votes |
|
public boolean connectToTopic() {
boolean r = true;
PartitionMetadata metadata = findLeader(brokerList, topic, partition);
if (metadata == null) {
r = false;
LOG.error("Can't find meta data for Topic: " + topic + " partition: " + partition
+ ". In fact, meta is null.");
}
else if (metadata.leader() == null) {
r = false;
LOG.error("Can't find meta data for Topic: " + topic + " partition: " + partition);
}
if (r) {
storeReplicaBrokers(metadata);
consumer = new SimpleConsumer(
metadata.leader().host(),
metadata.leader().port(),
10000, // timeout
64 * 1024, // buffersize
getClientName());
//scheduleCommitTimer();
}
return r;
}
Example 16
| Source File: SimpleConsumerExample.java From hermes with Apache License 2.0 | 5 votes |
|
/**
* @param a_oldLeader
* @param a_topic
* @param a_partition
* @param a_port
* @return String
* @throws Exception
* 找一个leader broker
*/
private String findNewLeader(String a_oldLeader, String a_topic, int a_partition, int a_port) throws Exception {
for (int i = 0; i < 3; i++) {
boolean goToSleep = false;
PartitionMetadata metadata = findLeader(m_replicaBrokers, a_port, a_topic, a_partition);
if (metadata == null) {
goToSleep = true;
} else if (metadata.leader() == null) {
goToSleep = true;
} else if (a_oldLeader.equalsIgnoreCase(metadata.leader().host()) && i == 0) {
// first time through if the leader hasn't changed give
// ZooKeeper a second to recover
// second time, assume the broker did recover before failover,
// or it was a non-Broker issue
//
goToSleep = true;
} else {
return metadata.leader().host();
}
if (goToSleep) {
try {
Thread.sleep(1000);
} catch (InterruptedException ie) {
}
}
}
System.out.println("Unable to find new leader after Broker failure. Exiting");
throw new Exception("Unable to find new leader after Broker failure. Exiting");
}
Example 17
| Source File: KafkaClusterTool.java From Scribengin with GNU Affero General Public License v3.0 | 5 votes |
|
Server findLeader(String topic, int partition) throws Exception {
KafkaTool kafkaTool = new KafkaTool("KafkaPartitionLeaderKiller", cluster.getZKConnect());
kafkaTool.connect();
TopicMetadata topicMeta = kafkaTool.findTopicMetadata(topic);
PartitionMetadata partitionMeta = findPartition(topicMeta, partition);
Broker partitionLeader = partitionMeta.leader();
Server kafkaServer = cluster.findKafkaServerByPort(partitionLeader.port());
System.out.println("Shutdown kafka server " + kafkaServer.getPort());
kafkaTool.close();
return kafkaServer;
}
