Java Code Examples for org.apache.kafka.common.requests.MetadataResponse#TopicMetadata
The following examples show how to use
org.apache.kafka.common.requests.MetadataResponse#TopicMetadata .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
| Source File: KafkaCruiseControlUtils.java From cruise-control with BSD 2-Clause "Simplified" License | 6 votes |
|
/**
* Generate a {@link MetadataResponseData} with the given information -- e.g. for creating bootstrap and test response.
*
* @param brokers Brokers in the cluster.
* @param clusterId Cluster Id.
* @param controllerId Controller Id.
* @param topicMetadataList Metadata list for the topics in the cluster.
* @return A {@link MetadataResponseData} with the given information.
*/
public static MetadataResponse prepareMetadataResponse(List<Node> brokers,
String clusterId,
int controllerId,
List<MetadataResponse.TopicMetadata> topicMetadataList) {
MetadataResponseData responseData = new MetadataResponseData();
responseData.setThrottleTimeMs(AbstractResponse.DEFAULT_THROTTLE_TIME);
brokers.forEach(broker -> responseData.brokers().add(
new MetadataResponseData.MetadataResponseBroker().setNodeId(broker.id())
.setHost(broker.host())
.setPort(broker.port())
.setRack(broker.rack())));
responseData.setClusterId(clusterId);
responseData.setControllerId(controllerId);
responseData.setClusterAuthorizedOperations(MetadataResponse.AUTHORIZED_OPERATIONS_OMITTED);
topicMetadataList.forEach(topicMetadata -> responseData.topics().add(prepareMetadataResponseTopic(topicMetadata)));
return new MetadataResponse(responseData);
}
Example 2
| Source File: KafkaCruiseControlUtils.java From cruise-control with BSD 2-Clause "Simplified" License | 6 votes |
|
private static MetadataResponseData.MetadataResponseTopic prepareMetadataResponseTopic(MetadataResponse.TopicMetadata topicMetadata) {
MetadataResponseData.MetadataResponseTopic metadataResponseTopic = new MetadataResponseData.MetadataResponseTopic();
metadataResponseTopic.setErrorCode(topicMetadata.error().code())
.setName(topicMetadata.topic())
.setIsInternal(topicMetadata.isInternal())
.setTopicAuthorizedOperations(topicMetadata.authorizedOperations());
for (MetadataResponse.PartitionMetadata partitionMetadata : topicMetadata.partitionMetadata()) {
metadataResponseTopic.partitions().add(
new MetadataResponseData.MetadataResponsePartition()
.setErrorCode(partitionMetadata.error().code())
.setPartitionIndex(partitionMetadata.partition())
.setLeaderId(partitionMetadata.leader() == null ? -1 : partitionMetadata.leader().id())
.setLeaderEpoch(partitionMetadata.leaderEpoch().orElse(RecordBatch.NO_PARTITION_LEADER_EPOCH))
.setReplicaNodes(partitionMetadata.replicas().stream().map(Node::id).collect(Collectors.toList()))
.setIsrNodes(partitionMetadata.isr().stream().map(Node::id).collect(Collectors.toList()))
.setOfflineReplicas(partitionMetadata.offlineReplicas().stream().map(Node::id).collect(Collectors.toList())));
}
return metadataResponseTopic;
}
Example 3
| Source File: MockKafka.java From kylin-on-parquet-v2 with Apache License 2.0 | 5 votes |
|
public MetadataResponse.TopicMetadata fetchTopicMeta(String topic) {
ZkClient zkClient = new ZkClient(zkConnection);
ZkUtils zkUtils = new ZkUtils(zkClient, zkConnection, false);
zkClient.setZkSerializer(new ZKStringSerializer());
MetadataResponse.TopicMetadata topicMetadata = AdminUtils.fetchTopicMetadataFromZk(topic, zkUtils);
zkClient.close();
return topicMetadata;
}
Example 4
| Source File: MockKafka.java From kylin with Apache License 2.0 | 5 votes |
|
public MetadataResponse.TopicMetadata fetchTopicMeta(String topic) {
ZkClient zkClient = new ZkClient(zkConnection);
ZkUtils zkUtils = new ZkUtils(zkClient, zkConnection, false);
zkClient.setZkSerializer(new ZKStringSerializer());
MetadataResponse.TopicMetadata topicMetadata = AdminUtils.fetchTopicMetadataFromZk(topic, zkUtils);
zkClient.close();
return topicMetadata;
}
Example 5
| Source File: KafkaPartitionMetricSampleAggregatorTest.java From cruise-control with BSD 2-Clause "Simplified" License | 4 votes |
|
@Test
public void testAggregateWithUpdatedCluster() throws NotEnoughValidWindowsException {
KafkaCruiseControlConfig config = new KafkaCruiseControlConfig(getLoadMonitorProperties());
Metadata metadata = getMetadata(Collections.singleton(TP));
KafkaPartitionMetricSampleAggregator
metricSampleAggregator = new KafkaPartitionMetricSampleAggregator(config, metadata);
populateSampleAggregator(NUM_WINDOWS + 1, MIN_SAMPLES_PER_WINDOW, metricSampleAggregator);
TopicPartition tp1 = new TopicPartition(TOPIC0 + "1", 0);
Cluster cluster = getCluster(Arrays.asList(TP, tp1));
List<MetadataResponse.TopicMetadata> topicMetadata = new ArrayList<>(2);
topicMetadata.add(new MetadataResponse.TopicMetadata(Errors.NONE,
TOPIC0,
false,
Collections.singletonList(new MetadataResponse.PartitionMetadata(
Errors.NONE, PARTITION, NODE_0,
Optional.of(RecordBatch.NO_PARTITION_LEADER_EPOCH),
Arrays.asList(nodes()), Arrays.asList(nodes()),
Collections.emptyList()))));
topicMetadata.add(new MetadataResponse.TopicMetadata(Errors.NONE,
TOPIC0 + "1",
false,
Collections.singletonList(new MetadataResponse.PartitionMetadata(
Errors.NONE, 0, NODE_0,
Optional.of(RecordBatch.NO_PARTITION_LEADER_EPOCH),
Arrays.asList(nodes()), Arrays.asList(nodes()),
Collections.emptyList()))));
MetadataResponse metadataResponse = KafkaCruiseControlUtils.prepareMetadataResponse(cluster.nodes(),
cluster.clusterResource().clusterId(),
MetadataResponse.NO_CONTROLLER_ID,
topicMetadata);
metadata.update(KafkaCruiseControlUtils.REQUEST_VERSION_UPDATE, metadataResponse, 1);
Map<PartitionEntity, ValuesAndExtrapolations> aggregateResult =
metricSampleAggregator.aggregate(cluster, Long.MAX_VALUE, new OperationProgress()).valuesAndExtrapolations();
// Partition "topic-0" should be valid in all NUM_WINDOW windows and Partition "topic1-0" should not since
// there is no sample for it.
assertEquals(1, aggregateResult.size());
assertEquals(NUM_WINDOWS, aggregateResult.get(PE).windows().size());
ModelCompletenessRequirements requirements =
new ModelCompletenessRequirements(1, 0.0, true);
MetricSampleAggregationResult<String, PartitionEntity> result =
metricSampleAggregator.aggregate(cluster, -1, Long.MAX_VALUE, requirements, new OperationProgress());
aggregateResult = result.valuesAndExtrapolations();
assertNotNull("tp1 should be included because includeAllTopics is set to true",
aggregateResult.get(new PartitionEntity(tp1)));
Map<Integer, Extrapolation> extrapolations = aggregateResult.get(new PartitionEntity(tp1)).extrapolations();
assertEquals(NUM_WINDOWS, extrapolations.size());
for (int i = 0; i < NUM_WINDOWS; i++) {
assertEquals(Extrapolation.NO_VALID_EXTRAPOLATION, extrapolations.get(i));
}
}
Example 6
| Source File: KafkaPartitionMetricSampleAggregatorTest.java From cruise-control with BSD 2-Clause "Simplified" License | 4 votes |
|
@Test
public void testAggregateWithPartitionExtrapolations() throws NotEnoughValidWindowsException {
KafkaCruiseControlConfig config = new KafkaCruiseControlConfig(getLoadMonitorProperties());
Metadata metadata = getMetadata(Collections.singleton(TP));
KafkaPartitionMetricSampleAggregator
metricSampleAggregator = new KafkaPartitionMetricSampleAggregator(config, metadata);
TopicPartition tp1 = new TopicPartition(TOPIC0, 1);
Cluster cluster = getCluster(Arrays.asList(TP, tp1));
PartitionEntity pe1 = new PartitionEntity(tp1);
List<MetadataResponse.PartitionMetadata> partitionMetadata =
Collections.singletonList(new MetadataResponse.PartitionMetadata(Errors.NONE, 1, NODE_0,
Optional.of(RecordBatch.NO_PARTITION_LEADER_EPOCH),
Arrays.asList(nodes()), Arrays.asList(nodes()),
Collections.emptyList()));
List<MetadataResponse.TopicMetadata> topicMetadata = Collections.singletonList(
new MetadataResponse.TopicMetadata(Errors.NONE, TOPIC0, false, partitionMetadata));
MetadataResponse metadataResponse = KafkaCruiseControlUtils.prepareMetadataResponse(cluster.nodes(),
cluster.clusterResource().clusterId(),
MetadataResponse.NO_CONTROLLER_ID,
topicMetadata);
metadata.update(KafkaCruiseControlUtils.REQUEST_VERSION_UPDATE, metadataResponse, 1);
populateSampleAggregator(NUM_WINDOWS + 1, MIN_SAMPLES_PER_WINDOW, metricSampleAggregator);
//Populate partition 1 but leave 1 hole at NUM_WINDOWS'th window.
CruiseControlUnitTestUtils.populateSampleAggregator(NUM_WINDOWS - 2, MIN_SAMPLES_PER_WINDOW,
metricSampleAggregator,
pe1,
0, WINDOW_MS,
KafkaMetricDef.commonMetricDef());
CruiseControlUnitTestUtils.populateSampleAggregator(2, MIN_SAMPLES_PER_WINDOW,
metricSampleAggregator,
pe1,
NUM_WINDOWS - 1, WINDOW_MS,
KafkaMetricDef.commonMetricDef());
MetricSampleAggregationResult<String, PartitionEntity> result =
metricSampleAggregator.aggregate(cluster, Long.MAX_VALUE, new OperationProgress());
assertEquals(2, result.valuesAndExtrapolations().size());
assertTrue(result.valuesAndExtrapolations().get(PE).extrapolations().isEmpty());
assertEquals(1, result.valuesAndExtrapolations().get(pe1).extrapolations().size());
assertTrue(result.valuesAndExtrapolations().get(pe1).extrapolations().containsKey(1));
assertEquals((NUM_WINDOWS - 1) * WINDOW_MS, result.valuesAndExtrapolations().get(pe1).window(1));
assertEquals(Extrapolation.AVG_ADJACENT, result.valuesAndExtrapolations().get(pe1).extrapolations().get(1));
}
