org.apache.kafka.common.requests.MetadataResponse Java Examples
The following examples show how to use
org.apache.kafka.common.requests.MetadataResponse.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
| Source File: KafkaTestEnvironmentImpl.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
|
@Override
public int getLeaderToShutDown(String topic) throws Exception {
ZkUtils zkUtils = getZkUtils();
try {
MetadataResponse.PartitionMetadata firstPart = null;
do {
if (firstPart != null) {
LOG.info("Unable to find leader. error code {}", firstPart.error().code());
// not the first try. Sleep a bit
Thread.sleep(150);
}
List<MetadataResponse.PartitionMetadata> partitionMetadata = AdminUtils.fetchTopicMetadataFromZk(topic, zkUtils).partitionMetadata();
firstPart = partitionMetadata.get(0);
}
while (firstPart.error().code() != 0);
return firstPart.leader().id();
} finally {
zkUtils.close();
}
}
Example #2
| Source File: KafkaTestEnvironmentImpl.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
|
@Override
public int getLeaderToShutDown(String topic) throws Exception {
ZkUtils zkUtils = getZkUtils();
try {
MetadataResponse.PartitionMetadata firstPart = null;
do {
if (firstPart != null) {
LOG.info("Unable to find leader. error code {}", firstPart.error().code());
// not the first try. Sleep a bit
Thread.sleep(150);
}
List<MetadataResponse.PartitionMetadata> partitionMetadata = AdminUtils.fetchTopicMetadataFromZk(topic, zkUtils).partitionMetadata();
firstPart = partitionMetadata.get(0);
}
while (firstPart.error().code() != 0);
return firstPart.leader().id();
} finally {
zkUtils.close();
}
}
Example #3
| Source File: BootstrapServerParam.java From kafka_book_demo with Apache License 2.0 | 6 votes |
|
public static MetadataResponse getMetadata(int throttleTimeMs) {
Node node = new Node(0, "localhost", 9093);
List<Node> brokers = Collections.singletonList(node);
int controllerId = 0;
String clusterId = "64PniqfkRHa4ASFUisNXrw";
List<Node> empty = new ArrayList<>();
PartitionMetadata pMeta1 = new PartitionMetadata(Errors.NONE, 0, node, brokers, brokers, empty);
PartitionMetadata pMeta2 = new PartitionMetadata(Errors.NONE, 1, node, brokers, brokers, empty);
PartitionMetadata pMeta3 = new PartitionMetadata(Errors.NONE, 2, node, brokers, brokers, empty);
PartitionMetadata pMeta4 = new PartitionMetadata(Errors.NONE, 3, node, brokers, brokers, empty);
List<PartitionMetadata> pMetaList = new ArrayList<>();
pMetaList.add(pMeta1);
pMetaList.add(pMeta2);
pMetaList.add(pMeta3);
pMetaList.add(pMeta4);
TopicMetadata tMeta1 = new TopicMetadata(Errors.NONE, topic, false, pMetaList);
List<TopicMetadata> tMetaList = new ArrayList<>();
tMetaList.add(tMeta1);
return new MetadataResponse(throttleTimeMs, brokers, clusterId, controllerId, tMetaList);
}
Example #4
| Source File: KafkaTestEnvironmentImpl.java From flink with Apache License 2.0 | 6 votes |
|
@Override
public int getLeaderToShutDown(String topic) throws Exception {
ZkUtils zkUtils = getZkUtils();
try {
MetadataResponse.PartitionMetadata firstPart = null;
do {
if (firstPart != null) {
LOG.info("Unable to find leader. error code {}", firstPart.error().code());
// not the first try. Sleep a bit
Thread.sleep(150);
}
List<MetadataResponse.PartitionMetadata> partitionMetadata = AdminUtils.fetchTopicMetadataFromZk(topic, zkUtils).partitionMetadata();
firstPart = partitionMetadata.get(0);
}
while (firstPart.error().code() != 0);
return firstPart.leader().id();
} finally {
zkUtils.close();
}
}
Example #5
| Source File: KafkaTestEnvironmentImpl.java From flink with Apache License 2.0 | 6 votes |
|
@Override
public int getLeaderToShutDown(String topic) throws Exception {
ZkUtils zkUtils = getZkUtils();
try {
MetadataResponse.PartitionMetadata firstPart = null;
do {
if (firstPart != null) {
LOG.info("Unable to find leader. error code {}", firstPart.error().code());
// not the first try. Sleep a bit
Thread.sleep(150);
}
List<MetadataResponse.PartitionMetadata> partitionMetadata = AdminUtils.fetchTopicMetadataFromZk(topic, zkUtils).partitionMetadata();
firstPart = partitionMetadata.get(0);
}
while (firstPart.error().code() != 0);
return firstPart.leader().id();
} finally {
zkUtils.close();
}
}
Example #6
| Source File: KafkaCruiseControlUtils.java From cruise-control with BSD 2-Clause "Simplified" License | 6 votes |
|
/**
* Generate a {@link MetadataResponseData} with the given information -- e.g. for creating bootstrap and test response.
*
* @param brokers Brokers in the cluster.
* @param clusterId Cluster Id.
* @param controllerId Controller Id.
* @param topicMetadataList Metadata list for the topics in the cluster.
* @return A {@link MetadataResponseData} with the given information.
*/
public static MetadataResponse prepareMetadataResponse(List<Node> brokers,
String clusterId,
int controllerId,
List<MetadataResponse.TopicMetadata> topicMetadataList) {
MetadataResponseData responseData = new MetadataResponseData();
responseData.setThrottleTimeMs(AbstractResponse.DEFAULT_THROTTLE_TIME);
brokers.forEach(broker -> responseData.brokers().add(
new MetadataResponseData.MetadataResponseBroker().setNodeId(broker.id())
.setHost(broker.host())
.setPort(broker.port())
.setRack(broker.rack())));
responseData.setClusterId(clusterId);
responseData.setControllerId(controllerId);
responseData.setClusterAuthorizedOperations(MetadataResponse.AUTHORIZED_OPERATIONS_OMITTED);
topicMetadataList.forEach(topicMetadata -> responseData.topics().add(prepareMetadataResponseTopic(topicMetadata)));
return new MetadataResponse(responseData);
}
Example #7
| Source File: KafkaCruiseControlUtils.java From cruise-control with BSD 2-Clause "Simplified" License | 6 votes |
|
private static MetadataResponseData.MetadataResponseTopic prepareMetadataResponseTopic(MetadataResponse.TopicMetadata topicMetadata) {
MetadataResponseData.MetadataResponseTopic metadataResponseTopic = new MetadataResponseData.MetadataResponseTopic();
metadataResponseTopic.setErrorCode(topicMetadata.error().code())
.setName(topicMetadata.topic())
.setIsInternal(topicMetadata.isInternal())
.setTopicAuthorizedOperations(topicMetadata.authorizedOperations());
for (MetadataResponse.PartitionMetadata partitionMetadata : topicMetadata.partitionMetadata()) {
metadataResponseTopic.partitions().add(
new MetadataResponseData.MetadataResponsePartition()
.setErrorCode(partitionMetadata.error().code())
.setPartitionIndex(partitionMetadata.partition())
.setLeaderId(partitionMetadata.leader() == null ? -1 : partitionMetadata.leader().id())
.setLeaderEpoch(partitionMetadata.leaderEpoch().orElse(RecordBatch.NO_PARTITION_LEADER_EPOCH))
.setReplicaNodes(partitionMetadata.replicas().stream().map(Node::id).collect(Collectors.toList()))
.setIsrNodes(partitionMetadata.isr().stream().map(Node::id).collect(Collectors.toList()))
.setOfflineReplicas(partitionMetadata.offlineReplicas().stream().map(Node::id).collect(Collectors.toList())));
}
return metadataResponseTopic;
}
Example #8
| Source File: KafkaTestEnvironmentImpl.java From flink with Apache License 2.0 | 6 votes |
|
@Override
public int getLeaderToShutDown(String topic) throws Exception {
ZkUtils zkUtils = getZkUtils();
try {
MetadataResponse.PartitionMetadata firstPart = null;
do {
if (firstPart != null) {
LOG.info("Unable to find leader. error code {}", firstPart.error().code());
// not the first try. Sleep a bit
Thread.sleep(150);
}
List<MetadataResponse.PartitionMetadata> partitionMetadata = AdminUtils.fetchTopicMetadataFromZk(topic, zkUtils).partitionMetadata();
firstPart = partitionMetadata.get(0);
}
while (firstPart.error().code() != 0);
return firstPart.leader().id();
} finally {
zkUtils.close();
}
}
Example #9
| Source File: KafkaTestEnvironmentImpl.java From flink with Apache License 2.0 | 6 votes |
|
@Override
public int getLeaderToShutDown(String topic) throws Exception {
ZkUtils zkUtils = getZkUtils();
try {
MetadataResponse.PartitionMetadata firstPart = null;
do {
if (firstPart != null) {
LOG.info("Unable to find leader. error code {}", firstPart.error().code());
// not the first try. Sleep a bit
Thread.sleep(150);
}
List<MetadataResponse.PartitionMetadata> partitionMetadata = AdminUtils.fetchTopicMetadataFromZk(topic, zkUtils).partitionMetadata();
firstPart = partitionMetadata.get(0);
}
while (firstPart.error().code() != 0);
return firstPart.leader().id();
} finally {
zkUtils.close();
}
}
Example #10
| Source File: KafkaApisTest.java From kop with Apache License 2.0 | 5 votes |
|
@Test(timeOut = 20000)
public void testBrokerHandleTopicMetadataRequest() throws Exception {
String topicName = "kopBrokerHandleTopicMetadataRequest";
int numberTopics = 5;
int numberPartitions = 6;
List<TopicPartition> topicPartitions = createTopics(topicName, numberTopics, numberPartitions);
List<String> kafkaTopics = getCreatedTopics(topicName, numberTopics);
KafkaHeaderAndRequest metadataRequest = createTopicMetadataRequest(kafkaTopics);
CompletableFuture<AbstractResponse> responseFuture = new CompletableFuture<>();
kafkaRequestHandler.handleTopicMetadataRequest(metadataRequest, responseFuture);
MetadataResponse metadataResponse = (MetadataResponse) responseFuture.get();
// verify all served by same broker : localhost:port
assertEquals(metadataResponse.brokers().size(), 1);
assertEquals(metadataResponse.brokers().iterator().next().host(), "localhost");
// check metadata response
Collection<TopicMetadata> topicMetadatas = metadataResponse.topicMetadata();
log.debug("a. dumpTopicMetadata: ");
topicMetadatas.forEach(topicMetadata -> {
log.debug(" topicMetadata: {}", topicMetadata);
log.debug("b. dumpPartitionMetadata: ");
topicMetadata.partitionMetadata().forEach(partition -> {
log.debug(" PartitionMetadata: {}", partition);
});
});
assertEquals(topicMetadatas.size(), numberTopics);
topicMetadatas.forEach(topicMetadata -> {
assertTrue(topicMetadata.topic().startsWith(topicName + "_"));
assertEquals(topicMetadata.partitionMetadata().size(), numberPartitions);
});
}
Example #11
| Source File: MockKafka.java From kylin-on-parquet-v2 with Apache License 2.0 | 5 votes |
|
public MetadataResponse.TopicMetadata fetchTopicMeta(String topic) {
ZkClient zkClient = new ZkClient(zkConnection);
ZkUtils zkUtils = new ZkUtils(zkClient, zkConnection, false);
zkClient.setZkSerializer(new ZKStringSerializer());
MetadataResponse.TopicMetadata topicMetadata = AdminUtils.fetchTopicMetadataFromZk(topic, zkUtils);
zkClient.close();
return topicMetadata;
}
Example #12
| Source File: MetadataClient.java From cruise-control with BSD 2-Clause "Simplified" License | 5 votes |
|
public MetadataClient(KafkaCruiseControlConfig config,
Metadata metadata,
long metadataTTL,
Time time) {
_metadataGeneration = new AtomicInteger(0);
_metadata = metadata;
_refreshMetadataTimeout = config.getLong(MonitorConfig.METADATA_MAX_AGE_CONFIG);
_time = time;
List<InetSocketAddress> addresses =
ClientUtils.parseAndValidateAddresses(config.getList(MonitorConfig.BOOTSTRAP_SERVERS_CONFIG),
ClientDnsLookup.DEFAULT);
Cluster bootstrapCluster = Cluster.bootstrap(addresses);
MetadataResponse metadataResponse = KafkaCruiseControlUtils.prepareMetadataResponse(bootstrapCluster.nodes(),
bootstrapCluster.clusterResource().clusterId(),
MetadataResponse.NO_CONTROLLER_ID,
Collections.emptyList());
_metadata.update(KafkaCruiseControlUtils.REQUEST_VERSION_UPDATE, metadataResponse, time.milliseconds());
ChannelBuilder channelBuilder = ClientUtils.createChannelBuilder(config, time);
NetworkClientProvider provider = config.getConfiguredInstance(MonitorConfig.NETWORK_CLIENT_PROVIDER_CLASS_CONFIG,
NetworkClientProvider.class);
_networkClient = provider.createNetworkClient(config.getLong(MonitorConfig.CONNECTIONS_MAX_IDLE_MS_CONFIG),
new Metrics(),
time,
"load-monitor",
channelBuilder,
_metadata,
config.getString(MonitorConfig.CLIENT_ID_CONFIG),
DEFAULT_MAX_IN_FLIGHT_REQUEST,
config.getLong(MonitorConfig.RECONNECT_BACKOFF_MS_CONFIG),
config.getLong(MonitorConfig.RECONNECT_BACKOFF_MS_CONFIG),
config.getInt(MonitorConfig.SEND_BUFFER_CONFIG),
config.getInt(MonitorConfig.RECEIVE_BUFFER_CONFIG),
config.getInt(MonitorConfig.REQUEST_TIMEOUT_MS_CONFIG),
true,
new ApiVersions());
_metadataTTL = metadataTTL;
}
Example #13
| Source File: MockKafka.java From kylin with Apache License 2.0 | 5 votes |
|
public MetadataResponse.TopicMetadata fetchTopicMeta(String topic) {
ZkClient zkClient = new ZkClient(zkConnection);
ZkUtils zkUtils = new ZkUtils(zkClient, zkConnection, false);
zkClient.setZkSerializer(new ZKStringSerializer());
MetadataResponse.TopicMetadata topicMetadata = AdminUtils.fetchTopicMetadataFromZk(topic, zkUtils);
zkClient.close();
return topicMetadata;
}
Example #14
| Source File: KafkaPartitionMetricSampleAggregatorTest.java From cruise-control with BSD 2-Clause "Simplified" License | 4 votes |
|
@Test
public void testAggregateWithUpdatedCluster() throws NotEnoughValidWindowsException {
KafkaCruiseControlConfig config = new KafkaCruiseControlConfig(getLoadMonitorProperties());
Metadata metadata = getMetadata(Collections.singleton(TP));
KafkaPartitionMetricSampleAggregator
metricSampleAggregator = new KafkaPartitionMetricSampleAggregator(config, metadata);
populateSampleAggregator(NUM_WINDOWS + 1, MIN_SAMPLES_PER_WINDOW, metricSampleAggregator);
TopicPartition tp1 = new TopicPartition(TOPIC0 + "1", 0);
Cluster cluster = getCluster(Arrays.asList(TP, tp1));
List<MetadataResponse.TopicMetadata> topicMetadata = new ArrayList<>(2);
topicMetadata.add(new MetadataResponse.TopicMetadata(Errors.NONE,
TOPIC0,
false,
Collections.singletonList(new MetadataResponse.PartitionMetadata(
Errors.NONE, PARTITION, NODE_0,
Optional.of(RecordBatch.NO_PARTITION_LEADER_EPOCH),
Arrays.asList(nodes()), Arrays.asList(nodes()),
Collections.emptyList()))));
topicMetadata.add(new MetadataResponse.TopicMetadata(Errors.NONE,
TOPIC0 + "1",
false,
Collections.singletonList(new MetadataResponse.PartitionMetadata(
Errors.NONE, 0, NODE_0,
Optional.of(RecordBatch.NO_PARTITION_LEADER_EPOCH),
Arrays.asList(nodes()), Arrays.asList(nodes()),
Collections.emptyList()))));
MetadataResponse metadataResponse = KafkaCruiseControlUtils.prepareMetadataResponse(cluster.nodes(),
cluster.clusterResource().clusterId(),
MetadataResponse.NO_CONTROLLER_ID,
topicMetadata);
metadata.update(KafkaCruiseControlUtils.REQUEST_VERSION_UPDATE, metadataResponse, 1);
Map<PartitionEntity, ValuesAndExtrapolations> aggregateResult =
metricSampleAggregator.aggregate(cluster, Long.MAX_VALUE, new OperationProgress()).valuesAndExtrapolations();
// Partition "topic-0" should be valid in all NUM_WINDOW windows and Partition "topic1-0" should not since
// there is no sample for it.
assertEquals(1, aggregateResult.size());
assertEquals(NUM_WINDOWS, aggregateResult.get(PE).windows().size());
ModelCompletenessRequirements requirements =
new ModelCompletenessRequirements(1, 0.0, true);
MetricSampleAggregationResult<String, PartitionEntity> result =
metricSampleAggregator.aggregate(cluster, -1, Long.MAX_VALUE, requirements, new OperationProgress());
aggregateResult = result.valuesAndExtrapolations();
assertNotNull("tp1 should be included because includeAllTopics is set to true",
aggregateResult.get(new PartitionEntity(tp1)));
Map<Integer, Extrapolation> extrapolations = aggregateResult.get(new PartitionEntity(tp1)).extrapolations();
assertEquals(NUM_WINDOWS, extrapolations.size());
for (int i = 0; i < NUM_WINDOWS; i++) {
assertEquals(Extrapolation.NO_VALID_EXTRAPOLATION, extrapolations.get(i));
}
}
Example #15
| Source File: KafkaPartitionMetricSampleAggregatorTest.java From cruise-control with BSD 2-Clause "Simplified" License | 4 votes |
|
@Test
public void testAggregateWithPartitionExtrapolations() throws NotEnoughValidWindowsException {
KafkaCruiseControlConfig config = new KafkaCruiseControlConfig(getLoadMonitorProperties());
Metadata metadata = getMetadata(Collections.singleton(TP));
KafkaPartitionMetricSampleAggregator
metricSampleAggregator = new KafkaPartitionMetricSampleAggregator(config, metadata);
TopicPartition tp1 = new TopicPartition(TOPIC0, 1);
Cluster cluster = getCluster(Arrays.asList(TP, tp1));
PartitionEntity pe1 = new PartitionEntity(tp1);
List<MetadataResponse.PartitionMetadata> partitionMetadata =
Collections.singletonList(new MetadataResponse.PartitionMetadata(Errors.NONE, 1, NODE_0,
Optional.of(RecordBatch.NO_PARTITION_LEADER_EPOCH),
Arrays.asList(nodes()), Arrays.asList(nodes()),
Collections.emptyList()));
List<MetadataResponse.TopicMetadata> topicMetadata = Collections.singletonList(
new MetadataResponse.TopicMetadata(Errors.NONE, TOPIC0, false, partitionMetadata));
MetadataResponse metadataResponse = KafkaCruiseControlUtils.prepareMetadataResponse(cluster.nodes(),
cluster.clusterResource().clusterId(),
MetadataResponse.NO_CONTROLLER_ID,
topicMetadata);
metadata.update(KafkaCruiseControlUtils.REQUEST_VERSION_UPDATE, metadataResponse, 1);
populateSampleAggregator(NUM_WINDOWS + 1, MIN_SAMPLES_PER_WINDOW, metricSampleAggregator);
//Populate partition 1 but leave 1 hole at NUM_WINDOWS'th window.
CruiseControlUnitTestUtils.populateSampleAggregator(NUM_WINDOWS - 2, MIN_SAMPLES_PER_WINDOW,
metricSampleAggregator,
pe1,
0, WINDOW_MS,
KafkaMetricDef.commonMetricDef());
CruiseControlUnitTestUtils.populateSampleAggregator(2, MIN_SAMPLES_PER_WINDOW,
metricSampleAggregator,
pe1,
NUM_WINDOWS - 1, WINDOW_MS,
KafkaMetricDef.commonMetricDef());
MetricSampleAggregationResult<String, PartitionEntity> result =
metricSampleAggregator.aggregate(cluster, Long.MAX_VALUE, new OperationProgress());
assertEquals(2, result.valuesAndExtrapolations().size());
assertTrue(result.valuesAndExtrapolations().get(PE).extrapolations().isEmpty());
assertEquals(1, result.valuesAndExtrapolations().get(pe1).extrapolations().size());
assertTrue(result.valuesAndExtrapolations().get(pe1).extrapolations().containsKey(1));
assertEquals((NUM_WINDOWS - 1) * WINDOW_MS, result.valuesAndExtrapolations().get(pe1).window(1));
assertEquals(Extrapolation.AVG_ADJACENT, result.valuesAndExtrapolations().get(pe1).extrapolations().get(1));
}
