org.apache.kafka.clients.consumer.ConsumerRebalanceListener Java Examples

@Override
public void subscribe(Collection<String> topics, ConsumerRebalanceListener callback) {
    ObjectExtension.requireNonNull(topics, "topics");
    ObjectExtension.requireNonNull(callback, "callback");

    runWithoutConcurrency(() -> {
        synchronized (_addRemoveLock) {
            removeNotUsed(topics);
            topics.forEach(t -> {
                ConsumerHolder consumerHolder = getOrAddConsumer(t);
                consumerHolder.getConsumer().subscribe(Arrays.asList(t), callback);
                consumerHolder.setConsumerRebalanceListener(callback);
                consumerHolder.setAssignments(null);
            });
        }
    });
}
 

@Test
public void fail() {
  when(kafkaConsumerFactory.create()).thenReturn(consumer);
  TopicPartition topicPartition = new TopicPartition(DUMMY_TOPIC_NAME, 0);
  List<ConsumerRecord<String, byte[]>> recordList = new ArrayList<>();
  byte[] randomBytes = new byte[1];
  for (int i = 0; i < 5; i++) {
    RANDOM.nextBytes(randomBytes);
    recordList.add(new ConsumerRecord<>(DUMMY_TOPIC_NAME, 0, i, "key",
        Arrays.copyOf(randomBytes, randomBytes.length)));
  }
  ConsumerRecords<String, byte[]> consumerRecords = new ConsumerRecords<>(
      Collections.singletonMap(topicPartition, recordList));
  when(consumer.poll(any(Duration.class))).thenReturn(consumerRecords);

  kafkaSpout.open(Collections.singletonMap(Config.TOPOLOGY_RELIABILITY_MODE,
      ATLEAST_ONCE.name()), topologyContext, collector);
  verify(consumer).subscribe(eq(Collections.singleton(DUMMY_TOPIC_NAME)),
      consumerRebalanceListenerArgumentCaptor.capture());
  ConsumerRebalanceListener consumerRebalanceListener =
      consumerRebalanceListenerArgumentCaptor.getValue();
  consumerRebalanceListener.onPartitionsAssigned(Collections.singleton(topicPartition));
  //poll the topic
  kafkaSpout.nextTuple();
  //emit all of the five records
  for (int i = 0; i < 5; i++) {
    kafkaSpout.nextTuple();
  }
  //ack came in out of order, second and third record fails
  kafkaSpout.ack(new KafkaSpout.ConsumerRecordMessageId(topicPartition, 4));
  kafkaSpout.ack(new KafkaSpout.ConsumerRecordMessageId(topicPartition, 0));
  kafkaSpout.fail(new KafkaSpout.ConsumerRecordMessageId(topicPartition, 1));
  kafkaSpout.ack(new KafkaSpout.ConsumerRecordMessageId(topicPartition, 3));
  kafkaSpout.fail(new KafkaSpout.ConsumerRecordMessageId(topicPartition, 2));
  //commit and poll
  kafkaSpout.nextTuple();
  verify(consumer).seek(topicPartition, 1);
  verify(consumer).commitAsync(Collections.singletonMap(topicPartition,
      new OffsetAndMetadata(1)), null);
}
 

@Override
public void subscribe(Collection<String> topics, ConsumerRebalanceListener callback) {
  Set<String> newSubscription = new HashSet<>(topics);
  // TODO: This is a hot fix for KAFKA-3664 and should be removed after the issue is fixed.
  commitSync();
  for (TopicPartition tp : _kafkaConsumer.assignment()) {
    if (!newSubscription.contains(tp.topic())) {
      _consumerRecordsProcessor.clear(tp);
    }
  }
  _consumerRebalanceListener.setUserListener(callback);
  _kafkaConsumer.subscribe(new ArrayList<>(topics), _consumerRebalanceListener);
}
 

@Override
public void subscribe(Pattern pattern, ConsumerRebalanceListener callback) {
  if (callback != null) {
    _consumerRebalanceListener.setUserListener(callback);
  }
  _kafkaConsumer.subscribe(pattern, _consumerRebalanceListener);
}
 

@Override
public void subscribe(Pattern pattern, ConsumerRebalanceListener callback) {
  try (
      @SuppressWarnings("unused") CloseableLock uLock = new CloseableLock(userLock);
      @SuppressWarnings("unused") CloseableLock srLock = new CloseableLock(delegateLock.readLock())
  ) {
    verifyOpen();
    subscribedTopics = null;
    rebalanceListener = callback;
    assignedPartitions = null;
    subscriptionPattern = pattern;

    delegate.subscribe(pattern, callback);
  }
}
 

@Override
public void subscribe(Collection<String> topics, ConsumerRebalanceListener callback) {
  try (
      @SuppressWarnings("unused") CloseableLock uLock = new CloseableLock(userLock);
      @SuppressWarnings("unused") CloseableLock srLock = new CloseableLock(delegateLock.readLock())
  ) {
    verifyOpen();
    subscribedTopics = topics;
    rebalanceListener = callback;
    assignedPartitions = null;
    subscriptionPattern = null;

    delegate.subscribe(topics, callback);
  }
}
 

@Test
public void shouldUnsubscribeIfKafkaConsumerIsAlreadySubscribed() throws Exception {
    // given
    when(kafkaConsumer.subscription()).thenReturn(newSet(KAFKA_TOPIC));

    // when
    endpoint.consumeUntil(fromHorizon(), shutdown());

    // then
    verify(kafkaConsumer).unsubscribe();
    verify(kafkaConsumer).subscribe(eq(singletonList(KAFKA_TOPIC)), any(ConsumerRebalanceListener.class));
}
 

@Test
public void nextTuple() {
  when(kafkaConsumerFactory.create()).thenReturn(consumer);
  ConsumerRecords<String, byte[]> consumerRecords = new ConsumerRecords<>(
      Collections.singletonMap(new TopicPartition(DUMMY_TOPIC_NAME, 0),
          Collections.singletonList(new ConsumerRecord<>(DUMMY_TOPIC_NAME, 0, 0,
              "key", new byte[]{0xF}))));
  when(consumer.poll(any(Duration.class))).thenReturn(consumerRecords);
  doReturn(Collections.singletonMap(new MetricName("name", "group", "description",
      Collections.singletonMap("name", "value")), metric)).when(consumer).metrics();
  when(metric.metricValue()).thenReturn("sample value");

  kafkaSpout.open(Collections.singletonMap(Config.TOPOLOGY_RELIABILITY_MODE,
      ATMOST_ONCE.name()), topologyContext, collector);
  verify(consumer).subscribe(eq(Collections.singleton(DUMMY_TOPIC_NAME)),
      consumerRebalanceListenerArgumentCaptor.capture());
  ConsumerRebalanceListener consumerRebalanceListener =
      consumerRebalanceListenerArgumentCaptor.getValue();
  TopicPartition topicPartition = new TopicPartition(DUMMY_TOPIC_NAME, 0);
  consumerRebalanceListener.onPartitionsAssigned(Collections.singleton(topicPartition));

  kafkaSpout.nextTuple();
  verify(consumer).commitAsync();
  verify(topologyContext).registerMetric(eq("name-group-name-value"),
      kafkaMetricDecoratorArgumentCaptor.capture(), eq(60));
  assertEquals("sample value",
      kafkaMetricDecoratorArgumentCaptor.getValue().getValueAndReset());

  kafkaSpout.nextTuple();
  verify(collector).emit(eq("default"), listArgumentCaptor.capture());
  assertEquals("key", listArgumentCaptor.getValue().get(0));
  assertArrayEquals(new byte[]{0xF}, (byte[]) listArgumentCaptor.getValue().get(1));
}
 

@Test
public void ack() {
  when(kafkaConsumerFactory.create()).thenReturn(consumer);
  TopicPartition topicPartition = new TopicPartition(DUMMY_TOPIC_NAME, 0);
  List<ConsumerRecord<String, byte[]>> recordList = new ArrayList<>();
  byte[] randomBytes = new byte[1];
  for (int i = 0; i < 5; i++) {
    RANDOM.nextBytes(randomBytes);
    recordList.add(new ConsumerRecord<>(DUMMY_TOPIC_NAME, 0, i, "key",
        Arrays.copyOf(randomBytes, randomBytes.length)));
  }
  ConsumerRecords<String, byte[]> consumerRecords = new ConsumerRecords<>(
      Collections.singletonMap(topicPartition, recordList));
  when(consumer.poll(any(Duration.class))).thenReturn(consumerRecords);

  kafkaSpout.open(Collections.singletonMap(Config.TOPOLOGY_RELIABILITY_MODE,
      ATLEAST_ONCE.name()), topologyContext, collector);
  verify(consumer).subscribe(eq(Collections.singleton(DUMMY_TOPIC_NAME)),
      consumerRebalanceListenerArgumentCaptor.capture());
  ConsumerRebalanceListener consumerRebalanceListener =
      consumerRebalanceListenerArgumentCaptor.getValue();
  consumerRebalanceListener.onPartitionsAssigned(Collections.singleton(topicPartition));
  //poll the topic
  kafkaSpout.nextTuple();
  //emit all of the five records
  for (int i = 0; i < 5; i++) {
    kafkaSpout.nextTuple();
  }
  //ack came in out of order and the third record is not acknowledged
  kafkaSpout.ack(new KafkaSpout.ConsumerRecordMessageId(topicPartition, 4));
  kafkaSpout.ack(new KafkaSpout.ConsumerRecordMessageId(topicPartition, 0));
  kafkaSpout.ack(new KafkaSpout.ConsumerRecordMessageId(topicPartition, 1));
  kafkaSpout.ack(new KafkaSpout.ConsumerRecordMessageId(topicPartition, 3));
  //commit and poll
  kafkaSpout.nextTuple();
  verify(consumer).commitAsync(Collections.singletonMap(topicPartition,
      new OffsetAndMetadata(2)), null);
}
 

@Test
public void activate() {
  when(kafkaConsumerFactory.create()).thenReturn(consumer);
  kafkaSpout.open(Collections.singletonMap(Config.TOPOLOGY_RELIABILITY_MODE,
      ATMOST_ONCE.name()), topologyContext, collector);
  verify(consumer).subscribe(eq(Collections.singleton(DUMMY_TOPIC_NAME)),
      consumerRebalanceListenerArgumentCaptor.capture());
  ConsumerRebalanceListener consumerRebalanceListener =
      consumerRebalanceListenerArgumentCaptor.getValue();
  TopicPartition topicPartition = new TopicPartition(DUMMY_TOPIC_NAME, 0);
  consumerRebalanceListener.onPartitionsAssigned(Collections.singleton(topicPartition));
  kafkaSpout.activate();
  verify(consumer).resume(Collections.singleton(topicPartition));
}
 

@Test
public void deactivate() {
  when(kafkaConsumerFactory.create()).thenReturn(consumer);
  kafkaSpout.open(Collections.singletonMap(Config.TOPOLOGY_RELIABILITY_MODE,
      ATMOST_ONCE.name()), topologyContext, collector);
  verify(consumer).subscribe(eq(Collections.singleton(DUMMY_TOPIC_NAME)),
      consumerRebalanceListenerArgumentCaptor.capture());
  ConsumerRebalanceListener consumerRebalanceListener =
      consumerRebalanceListenerArgumentCaptor.getValue();
  TopicPartition topicPartition = new TopicPartition(DUMMY_TOPIC_NAME, 0);
  consumerRebalanceListener.onPartitionsAssigned(Collections.singleton(topicPartition));
  kafkaSpout.deactivate();
  verify(consumer).pause(Collections.singleton(topicPartition));
}
 

@Override
public void subscribe(RebalanceHandler handler, SecorConfig config) {
    ConsumerRebalanceListener reBalanceListener = new SecorConsumerRebalanceListener(mKafkaConsumer, mZookeeperConnector, getSkipZookeeperOffsetSeek(config), config.getNewConsumerAutoOffsetReset(), handler);
    ;

    String[] subscribeList = config.getKafkaTopicList();
    if (subscribeList.length == 0 || Strings.isNullOrEmpty(subscribeList[0])) {
        mKafkaConsumer.subscribe(Pattern.compile(config.getKafkaTopicFilter()), reBalanceListener);
    } else {
        mKafkaConsumer.subscribe(Arrays.asList(subscribeList), reBalanceListener);
    }
}
 

public void setConsumerRebalanceListener(ConsumerRebalanceListener consumerRebalanceListener) {
    _consumerRebalanceListener = consumerRebalanceListener;
}
 

@Override
public void subscribe(Collection<String> collection, ConsumerRebalanceListener consumerRebalanceListener) {
}
 

@Override
public void subscribe(Pattern pattern, ConsumerRebalanceListener callback) {
  _delegate.subscribe(pattern, callback);
}
 

@Override
public void subscribe(Pattern pattern, ConsumerRebalanceListener consumerRebalanceListener) {
}
 

@Override
public void subscribe(Collection<String> topics, ConsumerRebalanceListener callback) {
  _delegate.subscribe(topics, callback);
}
 

@Override public void subscribe(Collection<String> topics, ConsumerRebalanceListener callback) {
  delegate.subscribe(topics, callback);
}
 

@Override public void subscribe(Pattern pattern, ConsumerRebalanceListener callback) {
  delegate.subscribe(pattern, callback);
}
 

public void setUserListener(ConsumerRebalanceListener userListener) {
  _userListener = userListener;
}
 

static ConsumerRebalanceListeners of(final ConsumerRebalanceListener... listeners) {
    return new ConsumerRebalanceListeners(listeners);
}
 

private ConsumerRebalanceListeners(final ConsumerRebalanceListener... listeners) {
    this.listeners = asList(listeners);
}
 

@Override
public void subscribe(Pattern pattern, ConsumerRebalanceListener listener) {
  consumer.subscribe(pattern, listener);
}
 

@Override
public void subscribe(Collection<String> topics, ConsumerRebalanceListener listener) {
  consumer.subscribe(topics, listener);
}
 

@Override
ConsumerRebalanceListener getRebalanceListener() {
    return new CommitTrackingProcessingRebalanceListener();
}
 

ConsumerRebalanceListener getRebalanceListener() {
    return new ProcessingRebalanceListener();
}
 

@Override
public void subscribe(Pattern pattern, ConsumerRebalanceListener callback) {
    kafkaConsumer.subscribe(pattern, callback);
}
 

@Override
public void subscribe(Collection<String> topics, ConsumerRebalanceListener callback) {
    kafkaConsumer.subscribe(topics, callback);
}
 

@Override
public void subscribe(Pattern pattern, ConsumerRebalanceListener callback) {
    throw new UnsupportedOperationException();
}
 

public ConsumerRebalanceListener getConsumerRebalanceListener() {
    return _consumerRebalanceListener;
}