org.apache.kafka.clients.consumer.KafkaConsumer#close

Source File: CompositeTransactionManagerKafkaImpl.java From microservices-transactions-tcc with Apache License 2.0

6 votes

@SuppressWarnings("unchecked")
@Override
public List<EntityCommand<?>> fetch(String txId) {
	List<EntityCommand<?>> transactionOperations = new ArrayList<EntityCommand<?>>();

	Map<String, Object> consumerConfigs = (Map<String, Object>)configuration.get("kafkaConsumerConfiguration");
	consumerConfigs.put(ConsumerConfig.GROUP_ID_CONFIG, UUID.randomUUID().toString());
	
	KafkaConsumer<String, String> kafkaConsumer = new KafkaConsumer<String, String>(consumerConfigs);
	kafkaConsumer.subscribe(Arrays.asList(txId));
	
	ConsumerRecords<String, String> records = kafkaConsumer.poll(kafkaConsumerPollTimeout);
	for (ConsumerRecord<String, String> record : records){
		LOG.info("offset = {}, key = {}, value = {}", record.offset(), record.key(), record.value());
		try {
			transactionOperations.add(serializer.readFromString(record.value()));
		} catch (SerializationFailedException e) {
			LOG.error("Unable to deserialize [{}] because of: {}", record.value(), e.getMessage());
		}
	}
	
	kafkaConsumer.close();
		
	return transactionOperations;
}

Source File: ConsumerThread.java From kafka-topic-exporter with Apache License 2.0

6 votes

@Override
public void run() {

    KafkaConsumer<String, String> consumer = new KafkaConsumer<>(props.getProperties());
    consumer.subscribe(props.getTopicsPattern(), new KteConsumerRebalanceListener(consumer));
    try {
        while (!Thread.interrupted()) {
            ConsumerRecords<String, String> records = consumer.poll(100);
            if (records.count() > 0)
            	LOG.info("Got records count: " + String.valueOf(records.count()));
            for (ConsumerRecord<String, String> record : records) {
                String topic = record.topic();
                if(props.get(PropertyConfig.Constants.KAKFA_CONSUMER_REMOVEPREFIX.key,null) != null) {
                    topic = topic.replaceFirst("^" + props.get(PropertyConfig.Constants.KAKFA_CONSUMER_REMOVEPREFIX.key), "");
                }
                collector.add(topic, record.value());
            }
        }
    }
    finally {
        LOG.info("Shutting down consumer");
        consumer.close();
    }
}

Source File: TestKafkaServiceImpl.java From kafka-eagle with Apache License 2.0

6 votes

public Map<TopicPartition, Long> getKafkaLogSize(String topic, Set<Integer> partitionids) {
	Properties props = new Properties();
	props.put(ConsumerConfig.GROUP_ID_CONFIG, Kafka.KAFKA_EAGLE_SYSTEM_GROUP);
	props.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, "127.0.0.1:9092");
	props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getCanonicalName());
	props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getCanonicalName());
	KafkaConsumer<String, String> consumer = new KafkaConsumer<>(props);
	Set<TopicPartition> tps = new HashSet<>();
	Map<Integer, Long> partitionOffset = new HashMap<Integer, Long>();
	for (int partitionid : partitionids) {
		TopicPartition tp = new TopicPartition(topic, partitionid);
		long offset = consumer.position(tp);
		partitionOffset.put(partitionid, offset);
	}

	System.out.println(partitionOffset.toString());

	if (consumer != null) {
		consumer.close();
	}
	return null;
}

Source File: KafkaSource.java From kylin with Apache License 2.0

6 votes

@Override
public StreamingTableSourceInfo load(String cubeName) {
    KylinConfig kylinConf = KylinConfig.getInstanceFromEnv();
    CubeInstance cube = CubeManager.getInstance(kylinConf).getCube(cubeName);
    String streamingTableName = cube.getRootFactTable();
    StreamingSourceConfig streamingSourceConfig = StreamingSourceConfigManager.getInstance(kylinConf)
            .getConfig(streamingTableName);

    String topicName = getTopicName(streamingSourceConfig.getProperties());
    Map<String, Object> conf = getKafkaConf(streamingSourceConfig.getProperties(), cube.getConfig());

    KafkaConsumer<String, String> kafkaConsumer = new KafkaConsumer<String, String>(conf);
    try {
        List<PartitionInfo> partitionInfos = kafkaConsumer.partitionsFor(topicName);
        List<Partition> kafkaPartitions = Lists.transform(partitionInfos, new Function<PartitionInfo, Partition>() {
            @Nullable
            @Override
            public Partition apply(@Nullable PartitionInfo input) {
                return new Partition(input.partition());
            }
        });
        return new StreamingTableSourceInfo(kafkaPartitions);
    } finally {
        kafkaConsumer.close();
    }
}

Source File: SimulateResultService.java From SkaETL with Apache License 2.0

6 votes

public List<SimulateData> readOutPut(String bootStrapServers, String maxRecords, String windowTime) {
    KafkaConsumer kafkaConsumer = kafkaUtils.kafkaConsumer("latest", bootStrapServers, "simulate");
    log.info("Subscribe Topic for {}", SIMULATE_OUTPUT);
    kafkaConsumer.subscribe(Arrays.asList(SIMULATE_OUTPUT), new Rebalancer());
    List<SimulateData> res = new ArrayList<>();
    long start = System.currentTimeMillis();
    try {
        while (checkWindow(start, Long.valueOf(windowTime), res.size(), Long.valueOf(maxRecords))) {
            ConsumerRecords<String, SimulateData> records = kafkaConsumer.poll(100);
            for (ConsumerRecord<String, SimulateData> record : records) {
                res.add(record.value());
            }
            log.info("Number item for read OutPut {}", res.size());
            kafkaConsumer.commitSync();
        }
    } catch (WakeupException e) {
        // Ignore exception if closing
        throw e;
    } catch (RuntimeException re) {
        log.error("RuntimeException {}", re);
    } finally {
        kafkaConsumer.close();
    }
    return res;
}

Source File: KafkaServiceImpl.java From kafka-eagle with Apache License 2.0

5 votes

/**
 * Get kafka 0.10.x topic history logsize.
 */
public Map<TopicPartition, Long> getKafkaLogSize(String clusterAlias, String topic, Set<Integer> partitionids) {
	Properties props = new Properties();
	props.put(ConsumerConfig.GROUP_ID_CONFIG, Kafka.KAFKA_EAGLE_SYSTEM_GROUP);
	props.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, getKafkaBrokerServer(clusterAlias));
	props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getCanonicalName());
	props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getCanonicalName());
	if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.sasl.enable")) {
		sasl(props, clusterAlias);
	}
	if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.ssl.enable")) {
		ssl(props, clusterAlias);
	}
	KafkaConsumer<String, String> consumer = new KafkaConsumer<>(props);
	Set<TopicPartition> tps = new HashSet<>();
	for (int partitionid : partitionids) {
		TopicPartition tp = new TopicPartition(topic, partitionid);
		tps.add(tp);
	}

	consumer.assign(tps);
	java.util.Map<TopicPartition, Long> endLogSize = consumer.endOffsets(tps);
	if (consumer != null) {
		consumer.close();
	}
	return endLogSize;
}

Source File: TopicVerification.java From ja-micro with Apache License 2.0

5 votes

public boolean verifyTopicsExist(String kafkaBrokers, Set<String> requiredTopics,
                                 boolean checkPartitionCounts) {
    Properties props = new Properties();
    props.put("bootstrap.servers", kafkaBrokers);
    props.put("group.id", UUID.randomUUID().toString());
    props.put("key.deserializer", StringDeserializer.class.getName());
    props.put("value.deserializer", StringDeserializer.class.getName());
    if (serviceProperties != null) {
        SaslConfigurator configurator = new SaslConfigurator();
        configurator.configureSasl(props, serviceProperties.getServiceName(), serviceProperties.getKafkaPassword());
    } else {
        logger.warn("TopicVerification was not initialized, SASL will not be supported for this connection");
    }
    KafkaConsumer consumer = new KafkaConsumer(props);
    try {
        @SuppressWarnings("unchecked")
        Map<String, List<PartitionInfo>> topics = consumer.listTopics();

        Set<Integer> partitionCount = new HashSet<>();
        for (String requiredTopic : requiredTopics) {
            List<PartitionInfo> partitions = topics.get(requiredTopic);
            if (partitions == null) {
                logger.info("Required kafka topic {} not present", requiredTopic);
                return false;
            }
            partitionCount.add(partitions.size());
        }
        if (checkPartitionCounts && partitionCount.size() > 1) {
            logger.warn("Partition count mismatch in topics {}",
                    Arrays.toString(requiredTopics.toArray()));
            return false;
        }
        return true;
    } finally {
        consumer.close();
    }
}

Source File: SchemaRegistryConsumer.java From blog with MIT License

5 votes

public static void main(String[] args) {

    /** TODO: 设置 Consumer 属性 */
    Properties properties = new Properties();
    /** TODO: Kafka 服务地址 */
    properties.put(
        ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "node-160:9092,node-161:9092,node-162:9092");
    /** TODO: Key 序列化类 */
    properties.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
    /** TODO: Value 序列化类 */
    properties.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, KafkaAvroDeserializer.class);
    /** TODO: Consumer 组 */
    properties.put(ConsumerConfig.GROUP_ID_CONFIG, "consumer_group_schema");

    /** TODO: 设置 schema.registry */
    properties.put("schema.registry.url", "http://node-160:8081");

    /** TODO: 创建 Consumer */
    KafkaConsumer<String, GenericRecord> consumer = new KafkaConsumer<>(properties);

    /** TODO: 订阅主题：可以使用 Pattern.compile("") 正则表达式 */
    consumer.subscribe(Arrays.asList("topic01"));

    /** TODO: 遍历消息队列 */
    try {
      while (true) {
        /** TODO: 设置间隔多长时间获取消息 */
        ConsumerRecords<String, GenericRecord> consumerRecords =
            consumer.poll(Duration.ofSeconds(1));
        consumerRecords.forEach(
            r ->
                System.out.printf(
                    "partition = %d, offset = %d, key = %s, value = %s%n",
                    r.partition(), r.offset(), r.key(), r.value()));
      }
    } finally {
      /** TODO: 关闭 Consumer */
      consumer.close();
    }
  }

Source File: KafkaDispatcherImpl.java From arcusplatform with Apache License 2.0

5 votes

private void stop(KafkaConsumer<?, ?> consumer) {
	try {
		// note: this will commit offsets if offset commit is enabled
		consumer.close();
	}
	catch(Exception e) {
		logger.warn("Error cleanly disconnecting from topic [{}]", topic, e);
	}
}

Source File: CaseController.java From skywalking with Apache License 2.0

5 votes

@Override
public void run() {
    Properties consumerProperties = new Properties();
    consumerProperties.put("bootstrap.servers", bootstrapServers);
    consumerProperties.put("group.id", "testGroup");
    consumerProperties.put("enable.auto.commit", "true");
    consumerProperties.put("auto.commit.interval.ms", "1000");
    consumerProperties.put("auto.offset.reset", "earliest");
    consumerProperties.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
    consumerProperties.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
    KafkaConsumer<String, String> consumer = new KafkaConsumer<>(consumerProperties);
    consumer.subscribe(Arrays.asList(topicName));
    int i = 0;
    while (i++ <= 10) {
        try {
            Thread.sleep(1 * 1000);
        } catch (InterruptedException e) {
        }

        ConsumerRecords<String, String> records = consumer.poll(100);

        if (!records.isEmpty()) {
            for (ConsumerRecord<String, String> record : records) {
                logger.info("header: {}", new String(record.headers()
                                                           .headers("TEST")
                                                           .iterator()
                                                           .next()
                                                           .value()));
                logger.info("offset = {}, key = {}, value = {}", record.offset(), record.key(), record.value());
            }
            break;
        }
    }

    consumer.close();
}

Source File: KafkaConsumerAnalysis.java From BigData-In-Practice with Apache License 2.0

5 votes

public static void main(String[] args) {
        Properties props = initConfig();
        KafkaConsumer<String, String> consumer = new KafkaConsumer<>(props);
        // 订阅主题
//        consumer.subscribe(Arrays.asList(topic));
        // 还可以使用正则表达式
        // consumer.subscribe(Pattern.compile("topic.test.*"));

        // 直接订阅分区
        List<TopicPartition> partitions = new ArrayList<>();
        // 查询指定主题的元数据信息
        List<PartitionInfo> infos = consumer.partitionsFor(topic);
        if (infos != null) {
            for (PartitionInfo info : infos) {
                // 订阅所有分区，也可以订阅一部分分区
                partitions.add(new TopicPartition(info.topic(), info.partition()));
            }
            consumer.assign(partitions);
        }
        try {
            while (isRunning.get()) {
                ConsumerRecords<String, String> records = consumer.poll(Duration.ofMillis(1000));
                for (ConsumerRecord<String, String> record : records) {
                    System.out.println(String.format("topic = %s, partition = %s, offset = %s",
                            record.topic(), record.partition(), record.offset()));
                    System.out.println(String.format("key = %s, value = %s", record.key(), record.value()));
                }
            }
        } catch (Exception e) {
            log.error("发生异常 ", e);
            // 取消订阅
            consumer.unsubscribe();
        } finally {
            consumer.close();
        }
    }

Source File: KafkaAvroSerDesWithKafkaServerTest.java From registry with Apache License 2.0

5 votes

private ConsumerRecords<String, Object> consumeMessage(String topicName, String bootstrapServers, String consumerGroup) {
    Map<String, Object> props = new HashMap<>();
    props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers);
    props.putAll(SCHEMA_REGISTRY_TEST_SERVER_CLIENT_WRAPPER.exportClientConf(true));
    props.put(ConsumerConfig.GROUP_ID_CONFIG, consumerGroup);
    props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "true");
    props.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG, "1000");
    props.put(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG, "30000");
    props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, KafkaAvroDeserializer.class.getName());
    props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, KafkaAvroDeserializer.class.getName());

    KafkaConsumer<String, Object> consumer = new KafkaConsumer<>(props);

    List<PartitionInfo> partitionInfos = consumer.partitionsFor(topicName);
    Collection<TopicPartition> partitions = new ArrayList<>();
    for (PartitionInfo partitionInfo : partitionInfos) {
        partitions.add(new TopicPartition(partitionInfo.topic(), partitionInfo.partition()));
    }
    LOG.info("partitions [{}]", partitions);
    LOG.info("subscribed topis: [{}] ", consumer.listTopics());

    consumer.assign(partitions);
    consumer.seekToBeginning(partitions);

    ConsumerRecords<String, Object> consumerRecords = null;
    int ct = 0;
    while (ct++ < 100 && (consumerRecords == null || consumerRecords.isEmpty())) {
        LOG.info("Polling for consuming messages");
        consumerRecords = consumer.poll(Duration.ofMillis(500));
    }
    consumer.commitSync();
    consumer.close();

    return consumerRecords;
}

Source File: Consumer.java From blog with MIT License

5 votes

public static void main(String[] args) {
  /** TODO: 设置 Consumer 属性 */
  Properties properties = new Properties();
  /** TODO: Kafka 服务地址 */
  properties.put(
      ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "node-160:9092,node-161:9092,node-162:9092");
  /** TODO: Key 序列化类 */
  properties.put(
      ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName());
  /** TODO: Value 序列化类 */
  properties.put(
      ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName());
  /** TODO: Consumer 组 */
  properties.put(ConsumerConfig.GROUP_ID_CONFIG, "consumer_group_2");

  /** TODO: 创建 Consumer */
  KafkaConsumer<String, String> consumer = new KafkaConsumer<>(properties);

  /** TODO: 订阅主题：可以使用 Pattern.compile("") 正则表达式 */
  consumer.subscribe(Arrays.asList("topic01"));

  /** TODO: 遍历消息队列 */
  try {
    while (true) {
      /** TODO: 设置间隔多长时间获取消息 */
      ConsumerRecords<String, String> consumerRecords = consumer.poll(Duration.ofSeconds(1));
      consumerRecords.forEach(
          r ->
              System.out.printf(
                  "partition = %d, offset = %d, key = %s, value = %s%n",
                  r.partition(), r.offset(), r.key(), r.value()));
    }
  } finally {
    /** TODO: 关闭 Consumer */
    consumer.close();
  }
}

Source File: CheckFlowLineHandler.java From DBus with Apache License 2.0

4 votes

@Override
public void check(BufferedWriter bw) throws Exception {

    KafkaConsumer<String, byte[]> consumerSecond = null;
    KafkaConsumer<String, byte[]> consumerThird = null;
    KafkaConsumer<String, byte[]> consumerFourth = null;

    try {
        bw.newLine();
        bw.write("check flow line start: ");
        bw.newLine();
        bw.write("============================================");
        bw.newLine();

        List<Object> listSecond = initConsumer("testdb", "second");
        consumerSecond = (KafkaConsumer<String, byte[]>) listSecond.get(0);
        long offsetSecond = (Long) listSecond.get(1);

        List<Object> listThird = initConsumer("testdb.testschema", "third");
        consumerThird = (KafkaConsumer<String, byte[]>) listThird.get(0);
        long offsetThird = (Long) listThird.get(1);

        List<Object> listFourth = initConsumer("testdb.testschema.result", "fourth");
        consumerFourth = (KafkaConsumer<String, byte[]>) listFourth.get(0);
        // long offsetFourth = (Long) listFourth.get(1);

        long time = System.currentTimeMillis();

        firstStep(bw, time);
        secondStep(bw, consumerSecond, offsetSecond);
        thirdStep(bw, consumerThird, offsetThird);
        fourthStep(bw, consumerFourth, time);
    } catch (Exception e) {
        throw e;
    } finally {
        if (consumerSecond != null) consumerSecond.close();
        if (consumerThird != null) consumerThird.close();
        if (consumerFourth != null) consumerFourth.close();
    }

}

Source File: AvroKafkaConsumer.java From blog with MIT License

4 votes

public static void main(String[] args) {

    /** TODO: 使用 Avro 解析默认 */
    Schema.Parser parser = new Schema.Parser();
    Schema schema = parser.parse(AvroKafkaProducer.USER_SCHEMA);

    /** TODO: 使用 Bijection 将对象序列化转成字节数组 */
    Injection<GenericRecord, byte[]> recordInjection = GenericAvroCodecs.toBinary(schema);

    /** TODO: 设置 Consumer 属性 */
    Properties properties = new Properties();
    /** TODO: Kafka 服务地址 */
    properties.put(
        ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "node-160:9092,node-161:9092,node-162:9092");
    /** TODO: Key 序列化类 */
    properties.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
    /** TODO: Value 序列化类 */
    properties.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class);
    /** TODO: Consumer 组 */
    properties.put(ConsumerConfig.GROUP_ID_CONFIG, "consumer_avro");

    /** TODO: 创建 Consumer */
    KafkaConsumer<String, byte[]> consumer = new KafkaConsumer<>(properties);

    /** TODO: 订阅 Topic */
    consumer.subscribe(Pattern.compile("^topic01*$"));
    try {
      while (true) {
        /** TODO: 设置间隔多长时间获取消息 */
        ConsumerRecords<String, byte[]> records = consumer.poll(Duration.ofSeconds(1));
        records.forEach(
            record -> {
              /** TODO: 反序列化字节数组 */
              GenericRecord genericRecord = recordInjection.invert(record.value()).get();
              System.out.printf(
                  "partition = %d, offset = %d, name = %s, age = %d %n",
                  record.partition(),
                  record.offset(),
                  genericRecord.get("name"),
                  genericRecord.get("age"));
            });
      }
    } finally {
      /** TODO: 关闭 Consumer */
      consumer.close();
    }
  }

Source File: MetricsServer.java From arcusplatform with Apache License 2.0

4 votes

@Override
protected void start() throws Exception {
   log.info("Starting metrics processing server...");

   Properties props = kafkaOpsConfig.toNuConsumerProperties();
   Deserializer<JsonObject> delegate = JSON.createDeserializer(JsonObject.class);
   KafkaConsumer<String, JsonObject> consumer = new KafkaConsumer<>(props, new StringDeserializer(), new org.apache.kafka.common.serialization.Deserializer<JsonObject>() {
      @Override
      public void configure(Map<String, ?> configs, boolean isKey) {
         // no-op
      }

      @Override
      public JsonObject deserialize(String topic, byte[] data) {
         try {
            return delegate.deserialize(data);
         }
         catch(Exception e) {
            log.warn("could not deserialize: ", new String(data,StandardCharsets.UTF_8));
            return null;
         }
      }

      @Override
      public void close() {
         // no-op
      }
   });
   try {
      log.info("starting metrics consumer...");
      consumer.subscribe(ImmutableSet.of(kafkaOpsConfig.getTopicMetrics()));
      while(true) {
         ConsumerRecords<String, JsonObject> records = consumer.poll(kafkaOpsConfig.getPollingTimeoutMs());
         if(!records.isEmpty()) {
            consume(records);
         }
      }
   }
   catch (Exception ex) {
      log.warn("exiting abnormally: {}", ex.getMessage(), ex);
   }
   finally {
      consumer.commitSync();
      consumer.close();
   }
}

Source File: PastReplicaStatsProcessor.java From doctorkafka with Apache License 2.0

4 votes

public void run() {
  KafkaConsumer<byte[], byte[]> kafkaConsumer = null;
  try {
    String brokers = KafkaUtils.getBrokers(zkUrl, securityProtocol);
    LOG.info("ZkUrl: {}, Brokers: {}", zkUrl, brokers);
    Properties props = new Properties();
    props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, brokers);
    props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "false");
    props.put(ConsumerConfig.GROUP_ID_CONFIG, "doctorkafka_" + topicPartition);
    props.put(KafkaUtils.KEY_DESERIALIZER,
        "org.apache.kafka.common.serialization.ByteArrayDeserializer");
    props.put(KafkaUtils.VALUE_DESERIALIZER,
        "org.apache.kafka.common.serialization.ByteArrayDeserializer");
    props.put(KafkaUtils.MAX_POLL_RECORDS, 2000);
    props.put("max.partition.fetch.bytes", 1048576 * 4);

    kafkaConsumer = new KafkaConsumer<>(props);
    Set<TopicPartition> topicPartitions = new HashSet<>();
    topicPartitions.add(topicPartition);
    kafkaConsumer.assign(topicPartitions);
    kafkaConsumer.seek(topicPartition, startOffset);

    ConsumerRecords<byte[], byte[]> records = null;
    while (kafkaConsumer.position(topicPartition) < endOffset) {
      records = kafkaConsumer.poll(100);
      for (ConsumerRecord<byte[], byte[]> record : records) {
        BrokerStats brokerStats = OperatorUtil.deserializeBrokerStats(record);
        if (brokerStats == null || brokerStats.getName() == null) {
          OpenTsdbMetricConverter.incr(DoctorKafkaMetrics.MESSAGE_DESERIALIZE_ERROR, 1);
          continue;
        }
        replicaStatsManager.update(brokerStats);
      }
    }
  } catch (Exception e) {
    LOG.error("Exception in processing brokerstats", e);
  } finally {
    if (kafkaConsumer != null) {
      kafkaConsumer.close();
    }
  }
}

Source File: DuplicatePublishingDetector.java From light-eventuate-4j with Apache License 2.0

4 votes

private Optional<BinlogFileOffset> fetchMaxOffsetFor(String destinationTopic) {
  String subscriberId = "duplicate-checker-" + destinationTopic + "-" + System.currentTimeMillis();
  Properties consumerProperties = ConsumerPropertiesFactory.makeConsumerProperties(config, subscriberId);
  KafkaConsumer<String, String> consumer = new KafkaConsumer<>(consumerProperties);

  List<PartitionInfo> partitions = EventuateKafkaConsumer.verifyTopicExistsBeforeSubscribing(consumer, destinationTopic);

  List<TopicPartition> topicPartitionList = partitions.stream().map(p -> new TopicPartition(destinationTopic, p.partition())).collect(toList());
  consumer.assign(topicPartitionList);
  consumer.poll(0);

  logger.info("Seeking to end");

  try {
    consumer.seekToEnd(topicPartitionList);
  } catch (IllegalStateException e) {
    logger.error("Error seeking " + destinationTopic, e);
    return Optional.empty();
  }
  List<PartitionOffset> positions = topicPartitionList.stream()
          .map(tp -> new PartitionOffset(tp.partition(), consumer.position(tp) - 1))
          .filter(po -> po.offset >= 0)
          .collect(toList());

  logger.info("Seeking to positions=" + positions);

  positions.forEach(po -> {
    consumer.seek(new TopicPartition(destinationTopic, po.partition), po.offset);
  });

  logger.info("Polling for records");

  List<ConsumerRecord<String, String>> records = new ArrayList<>();
  while (records.size()<positions.size()) {
    ConsumerRecords<String, String> consumerRecords = consumer.poll(1000);
    consumerRecords.forEach(records::add);
  }

  logger.info("Got records: {}", records.size());
  Optional<BinlogFileOffset> max = StreamSupport.stream(records.spliterator(), false).map(record -> {
    logger.info(String.format("got record: %s %s %s", record.partition(), record.offset(), record.value()));
    return JSonMapper.fromJson(record.value(), PublishedEvent.class).getBinlogFileOffset();
  }).filter(binlogFileOffset -> binlogFileOffset!=null).max((blfo1, blfo2) -> blfo1.isSameOrAfter(blfo2) ? 1 : -1);
  consumer.close();
  return max;
}

Source File: KafkaService.java From cerberus-source with GNU General Public License v3.0

4 votes

@SuppressWarnings("unchecked")
@Override
public AnswerItem<Map<TopicPartition, Long>> seekEvent(String topic, String bootstrapServers,
        List<AppServiceHeader> serviceHeader) throws InterruptedException, ExecutionException {

    MessageEvent message = new MessageEvent(MessageEventEnum.ACTION_SUCCESS_CALLSERVICE_SEARCHKAFKA);
    AnswerItem<Map<TopicPartition, Long>> result = new AnswerItem<>();

    KafkaConsumer consumer = null;

    try {

        Properties props = new Properties();
        serviceHeader.add(factoryAppServiceHeader.create(null, "bootstrap.servers", bootstrapServers, "Y", 0, "", "", null, "", null));
        serviceHeader.add(factoryAppServiceHeader.create(null, "enable.auto.commit", "false", "Y", 0, "", "", null, "", null));
        serviceHeader.add(factoryAppServiceHeader.create(null, "max.poll.records", "10", "Y", 0, "", "", null, "", null));
        serviceHeader.add(factoryAppServiceHeader.create(null, "key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer", "Y", 0, "", "", null, "", null));
        serviceHeader.add(factoryAppServiceHeader.create(null, "value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer", "Y", 0, "", "", null, "", null));

        for (AppServiceHeader object : serviceHeader) {
            if (StringUtil.parseBoolean(object.getActive())) {
                props.put(object.getKey(), object.getValue());
            }
        }

        LOG.info("Open Consumer : " + getKafkaConsumerKey(topic, bootstrapServers));
        consumer = new KafkaConsumer<>(props);

        //Get a list of the topics' partitions
        List<PartitionInfo> partitionList = consumer.partitionsFor(topic);

        if (partitionList == null) {

            message = new MessageEvent(MessageEventEnum.ACTION_FAILED_CALLSERVICE_SEEKKAFKA);
            message.setDescription(message.getDescription().replace("%EX%", "Maybe Topic does not exist.").replace("%TOPIC%", topic).replace("%HOSTS%", bootstrapServers));

        } else {

            List<TopicPartition> topicPartitionList = partitionList.stream().map(info -> new TopicPartition(topic, info.partition())).collect(Collectors.toList());
            //Assign all the partitions to this consumer
            consumer.assign(topicPartitionList);
            consumer.seekToEnd(topicPartitionList); //default to latest offset for all partitions

            HashMap<TopicPartition, Long> valueResult = new HashMap<>();

            Map<TopicPartition, Long> partitionOffset = consumer.endOffsets(topicPartitionList);

            result.setItem(partitionOffset);

        }

    } catch (Exception ex) {
        message = new MessageEvent(MessageEventEnum.ACTION_FAILED_CALLSERVICE_SEEKKAFKA);
        message.setDescription(message.getDescription().replace("%EX%", ex.toString()).replace("%TOPIC%", topic).replace("%HOSTS%", bootstrapServers));
        LOG.debug(ex, ex);
    } finally {
        if (consumer != null) {
            consumer.close();
            LOG.info("Closed Consumer : " + getKafkaConsumerKey(topic, bootstrapServers));
        } else {
            LOG.info("Consumer not opened : " + getKafkaConsumerKey(topic, bootstrapServers));
        }
    }
    result.setResultMessage(message);
    return result;
}

Source File: SingerHeartbeatTest.java From singer with Apache License 2.0

4 votes

void testHeartBeat() {
  int numReceivedHeartbeats = 0;

  SingerTestHelper.createSingerConfig(
      singerConfigDir,
      singerConfigConfDir,
      singerDataDir,
      "singer_test_event",
      100,
      "singer_test_event",
      "",
      heartbeatIntervalInMilliSeconds / 1000,
      heartbeatTopic);

  SingerTestHelper.createSingerConfigLConfFile(
      singerConfigConfDir,
      "singer.test2.properties",
      singerDataDir,
      "singer_test_event_2",
      100,
      "singer_test_event",
      "");

  Process
      singerProc =
      SingerTestHelper
          .startSingerProcess(singerBinaryDir, singerConfigDir, SingerHeartbeatTest.class);

  File outputDir = new File(singerDataDir);
  ThriftLoggerFactory.initialize(outputDir, 100);

  SingerTestHelper.createLogStream(singerDataDir, "singer_test_event", 100, 500);
  SingerTestHelper.createLogStream(singerDataDir, "singer_test_event_2", 100, 500);

  SingerOutputRetriever outputRetriever = new SingerOutputRetriever(singerProc.getErrorStream());
  Thread outThread = new Thread(outputRetriever);
  outThread.start();

  try {
    Thread.sleep(20 * 1000);

    Properties properties = SingerTestHelper.createKafkaConsumerConfig();
    KafkaConsumer<byte[], byte[]> kafkaConsumer = new KafkaConsumer<>(properties);
    kafkaConsumer.subscribe(Arrays.asList(heartbeatTopic));

    SingerStatus status = null;

    for (int i = 0; i < numHeartbeats; i++) {
      Thread.sleep(heartbeatIntervalInMilliSeconds);

      String hostName = SingerUtils.getHostname();
      System.out.println("Fetching heartbeat messages from " + hostName + " : ");

      ConsumerRecords<byte[], byte[]> records = kafkaConsumer.poll(Duration.ofMillis(500L));
      for (ConsumerRecord<byte[], byte[]> record : records) {
        String msg = new String(record.value());
        status = new Gson().fromJson(msg, SingerStatus.class);

        if (System.currentTimeMillis() - status.getTimestamp() > heartbeatIntervalInMilliSeconds
                || !status.hostName.equals(hostName)) {
          System.out.println(msg);
          status = new Gson().fromJson(msg, SingerStatus.class);
          kafkaConsumer.commitSync();
        }

        System.out.println(msg);
        kafkaConsumer.commitSync();
        numReceivedHeartbeats++;
        assert (msg.contains("data.test"));
        assert (msg.contains("singer.test2"));
      }
    }
    kafkaConsumer.close();
  } catch (Exception e) {
    e.printStackTrace();
    assert (false);
  } finally {
    singerProc.destroy();
  }

  assert (numReceivedHeartbeats == numHeartbeats);
}

Java Code Examples for org.apache.kafka.clients.consumer.KafkaConsumer#close()