org.apache.kafka.common.serialization.BytesDeserializer Java Examples

The following examples show how to use org.apache.kafka.common.serialization.BytesDeserializer. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: KafkaAdminClientTest.java    From common-kafka with Apache License 2.0 6 votes vote down vote up 
@Test
public void getConsumerGroupSummary() {
    client.createTopic(testName.getMethodName(), 1, 1);

    Properties properties = new Properties();
    properties.putAll(KafkaTests.getProps());
    properties.setProperty(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, BytesDeserializer.class.getName());
    properties.setProperty(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, BytesDeserializer.class.getName());
    properties.setProperty(ConsumerConfig.GROUP_ID_CONFIG, testName.getMethodName());
    properties.setProperty(ConsumerConfig.CLIENT_ID_CONFIG, testName.getMethodName() + "-client-id");

    try (Consumer<Object, Object> consumer = new KafkaConsumer<>(properties)) {
        consumer.subscribe(Arrays.asList(testName.getMethodName()));
        consumer.poll(Duration.ofSeconds(5L));

        AdminClient.ConsumerGroupSummary summary = client.getConsumerGroupSummary(testName.getMethodName());
        assertThat("Expected only 1 consumer summary when getConsumerGroupSummaries(" + testName.getMethodName() + ")",
                convertToJavaSet(summary.consumers().get().iterator()).size(), is(1));

        assertThat(summary.state(), is(notNullValue()));
        assertThat(summary.coordinator(), is(notNullValue()));
        assertThat(summary.assignmentStrategy(), is(notNullValue()));
    }
}
Example #2
Source File: KafkaAdminClientTest.java    From common-kafka with Apache License 2.0 6 votes vote down vote up 
@Test(expected = UnsupportedOperationException.class)
public void getConsumerGroupSummaries_immutable() {
    client.createTopic(testName.getMethodName(), 1, 1);

    Properties properties = new Properties();
    properties.putAll(KafkaTests.getProps());
    properties.setProperty(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, BytesDeserializer.class.getName());
    properties.setProperty(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, BytesDeserializer.class.getName());
    properties.setProperty(ConsumerConfig.GROUP_ID_CONFIG, testName.getMethodName());
    properties.setProperty(ConsumerConfig.CLIENT_ID_CONFIG, testName.getMethodName() + "-client-id");

    try (Consumer<Object, Object> consumer = new KafkaConsumer<>(properties)) {
        consumer.subscribe(Arrays.asList(testName.getMethodName()));
        consumer.poll(Duration.ofSeconds(5L));

        client.getConsumerGroupSummaries(testName.getMethodName()).clear();
    }
}
Example #3
Source File: KafkaAdminClientTest.java    From common-kafka with Apache License 2.0 6 votes vote down vote up 
@Test
public void getConsumerGroupAssignments() {
    client.createTopic(testName.getMethodName(), 1, 1);

    Properties properties = new Properties();
    properties.putAll(KafkaTests.getProps());
    properties.setProperty(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, BytesDeserializer.class.getName());
    properties.setProperty(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, BytesDeserializer.class.getName());
    properties.setProperty(ConsumerConfig.GROUP_ID_CONFIG, testName.getMethodName());
    properties.setProperty(ConsumerConfig.CLIENT_ID_CONFIG, testName.getMethodName() + "-client-id");

    try (Consumer<Object, Object>  consumer = new KafkaConsumer<>(properties)) {
        consumer.subscribe(Arrays.asList(testName.getMethodName()));
        consumer.poll(Duration.ofSeconds(5L));

        assertThat(client.getConsumerGroupAssignments(testName.getMethodName()),
                is(Collections.singletonMap(new TopicPartition(testName.getMethodName(), 0),
                        testName.getMethodName() + "-client-id")));
    }
}
Example #4
Source File: KafkaAdminClientTest.java    From common-kafka with Apache License 2.0 6 votes vote down vote up 
@Test(expected = UnsupportedOperationException.class)
public void getConsumerGroupAssignments_immutable() {
    client.createTopic(testName.getMethodName(), 1, 1);

    Properties properties = new Properties();
    properties.putAll(KafkaTests.getProps());
    properties.setProperty(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, BytesDeserializer.class.getName());
    properties.setProperty(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, BytesDeserializer.class.getName());
    properties.setProperty(ConsumerConfig.GROUP_ID_CONFIG, testName.getMethodName());
    properties.setProperty(ConsumerConfig.CLIENT_ID_CONFIG, testName.getMethodName() + "-client-id");

    try (Consumer<Object, Object> consumer = new KafkaConsumer<>(properties)) {
        consumer.subscribe(Arrays.asList(testName.getMethodName()));
        consumer.poll(Duration.ofSeconds(5L));

        client.getConsumerGroupAssignments(testName.getMethodName()).clear();
    }
}
Example #5
Source File: ConsumerOffsetClient.java    From common-kafka with Apache License 2.0 6 votes vote down vote up 
private static Consumer<Object, Object> getConsumer(Properties properties) {
    if (properties == null)
        throw new IllegalArgumentException("properties cannot be null");

    Properties consumerProperties = new Properties();
    consumerProperties.putAll(properties);

    // Ensure the serializer configuration is set though its not needed
    consumerProperties.setProperty(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, BytesDeserializer.class.getName());
    consumerProperties.setProperty(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, BytesDeserializer.class.getName());

    String group = consumerProperties.getProperty(ConsumerConfig.GROUP_ID_CONFIG);

    // Add some random consumer group name to avoid any issues
    if (group == null)
        consumerProperties.put(ConsumerConfig.GROUP_ID_CONFIG, "kafka-consumer-offset-client-" + UUID.randomUUID());

    return new KafkaConsumer<>(consumerProperties);
}
Example #6
Source File: DataLoaderConfig.java    From kafka-webview with MIT License 6 votes vote down vote up 
/**
 * Creates default message formats.
 */
private void createDefaultMessageFormats() {
    final Map<String, String> defaultFormats = new HashMap<>();
    defaultFormats.put("Short", ShortDeserializer.class.getName());
    defaultFormats.put("ByteArray", ByteArrayDeserializer.class.getName());
    defaultFormats.put("Bytes", BytesDeserializer.class.getName());
    defaultFormats.put("Double", DoubleDeserializer.class.getName());
    defaultFormats.put("Float", FloatDeserializer.class.getName());
    defaultFormats.put("Integer", IntegerDeserializer.class.getName());
    defaultFormats.put("Long", LongDeserializer.class.getName());
    defaultFormats.put("String", StringDeserializer.class.getName());
    defaultFormats.put("Bytes (Hex Encoded)", BytesToHexDeserializer.class.getName());

    // Create if needed.
    for (final Map.Entry<String, String> entry : defaultFormats.entrySet()) {
        MessageFormat messageFormat = messageFormatRepository.findByName(entry.getKey());
        if (messageFormat == null) {
            messageFormat = new MessageFormat();
        }
        messageFormat.setName(entry.getKey());
        messageFormat.setClasspath(entry.getValue());
        messageFormat.setJar("n/a");
        messageFormat.setDefaultFormat(true);
        messageFormatRepository.save(messageFormat);
    }
}
Example #7
Source File: KafkaProducerInterceptorWrapper.java    From pulsar with Apache License 2.0 6 votes vote down vote up 
static Deserializer getDeserializer(Serializer serializer) {
    if (serializer instanceof StringSerializer) {
        return new StringDeserializer();
    } else if (serializer instanceof LongSerializer) {
        return new LongDeserializer();
    } else if (serializer instanceof IntegerSerializer) {
        return new IntegerDeserializer();
    } else if (serializer instanceof DoubleSerializer) {
        return new DoubleDeserializer();
    } else if (serializer instanceof BytesSerializer) {
        return new BytesDeserializer();
    } else if (serializer instanceof ByteBufferSerializer) {
        return new ByteBufferDeserializer();
    } else if (serializer instanceof ByteArraySerializer) {
        return new ByteArrayDeserializer();
    } else {
        throw new IllegalArgumentException(serializer.getClass().getName() + " is not a valid or supported subclass of org.apache.kafka.common.serialization.Serializer.");
    }
}
Example #8
Source File: KafkaProducerInterceptorWrapperTest.java    From pulsar with Apache License 2.0 6 votes vote down vote up 
@DataProvider(name = "serializers")
public Object[][] serializers() {
    return new Object[][] {
        {
            new StringSerializer(), StringDeserializer.class
        },
        {
            new LongSerializer(), LongDeserializer.class
        },
        {
            new IntegerSerializer(), IntegerDeserializer.class,
        },
        {
            new DoubleSerializer(), DoubleDeserializer.class,
        },
        {
            new BytesSerializer(), BytesDeserializer.class
        },
        {
            new ByteBufferSerializer(), ByteBufferDeserializer.class
        },
        {
            new ByteArraySerializer(), ByteArrayDeserializer.class
        }
    };
}
Example #9
Source File: TopicStreamWriter.java    From ksql-fork-with-deep-learning-function with Apache License 2.0 5 votes vote down vote up 
public TopicStreamWriter(
    SchemaRegistryClient schemaRegistryClient,
    Map<String, Object> consumerProperties,
    String topicName,
    long interval,
    long disconnectCheckInterval,
    boolean fromBeginning
) {
  this.schemaRegistryClient = schemaRegistryClient;
  this.topicName = topicName;
  this.messagesWritten = 0;

  this.disconnectCheckInterval = disconnectCheckInterval;

  this.topicConsumer = new KafkaConsumer<>(
      consumerProperties,
      new StringDeserializer(),
      new BytesDeserializer()
  );

  List<TopicPartition> topicPartitions = topicConsumer.partitionsFor(topicName)
      .stream()
      .map(partitionInfo -> new TopicPartition(partitionInfo.topic(), partitionInfo.partition()))
      .collect(Collectors.toList());
  topicConsumer.assign(topicPartitions);

  if (fromBeginning) {
    topicConsumer.seekToBeginning(topicPartitions);
  }

  this.interval = interval;
}
Example #10
Source File: KafkaAdminClientTest.java    From common-kafka with Apache License 2.0 5 votes vote down vote up 
@Test
public void getConsumerGroupSummaries() {
    client.createTopic(testName.getMethodName(), 1, 1);

    Properties properties = new Properties();
    properties.putAll(KafkaTests.getProps());
    properties.setProperty(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, BytesDeserializer.class.getName());
    properties.setProperty(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, BytesDeserializer.class.getName());
    properties.setProperty(ConsumerConfig.GROUP_ID_CONFIG, testName.getMethodName());
    properties.setProperty(ConsumerConfig.CLIENT_ID_CONFIG, testName.getMethodName() + "-client-id");

    try (Consumer<Object, Object> consumer = new KafkaConsumer<>(properties)) {
        consumer.subscribe(Arrays.asList(testName.getMethodName()));
        consumer.poll(Duration.ofSeconds(5L));

        Collection<AdminClient.ConsumerSummary> summaries = client.getConsumerGroupSummaries(testName.getMethodName());
        assertThat("Expected only 1 consumer summary when getConsumerGroupSummaries(" + testName.getMethodName() + ")",
                summaries.size(), is(1));

        AdminClient.ConsumerSummary summary = summaries.iterator().next();

        Collection<TopicPartition> assignments = convertToJavaSet(summary.assignment().iterator());
        assertThat("Expected consumer assignment to have single partition", assignments.size(), is(1));
        assertThat(assignments.iterator().next(), is(new TopicPartition(testName.getMethodName(), 0)));
        assertThat(summary.clientId(), is(testName.getMethodName() + "-client-id"));
    }
}
Example #11
Source File: KafkaPartitionLevelConnectionHandler.java    From incubator-pinot with Apache License 2.0 5 votes vote down vote up 
public KafkaPartitionLevelConnectionHandler(String clientId, StreamConfig streamConfig, int partition) {
  _config = new KafkaPartitionLevelStreamConfig(streamConfig);
  _clientId = clientId;
  _partition = partition;
  _topic = _config.getKafkaTopicName();
  Properties consumerProp = new Properties();
  consumerProp.putAll(streamConfig.getStreamConfigsMap());
  consumerProp.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, _config.getBootstrapHosts());
  consumerProp.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName());
  consumerProp.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, BytesDeserializer.class.getName());
  _consumer = new KafkaConsumer<>(consumerProp);
  _topicPartition = new TopicPartition(_topic, _partition);
  _consumer.assign(Collections.singletonList(_topicPartition));
}
Example #12
Source File: TestProperties.java    From kafka-workers with Apache License 2.0 4 votes vote down vote up 
public static Properties workersProperties() {
    return workersProperties(BytesDeserializer.class, BytesDeserializer.class, "test-topic");
}
Example #13
Source File: KafkaStreamLevelConsumerManager.java    From incubator-pinot with Apache License 2.0 4 votes vote down vote up 
public static KafkaConsumer acquireKafkaConsumerForConfig(KafkaStreamLevelStreamConfig kafkaStreamLevelStreamConfig) {
  final ImmutableTriple<String, String, String> configKey =
      new ImmutableTriple<>(kafkaStreamLevelStreamConfig.getKafkaTopicName(), kafkaStreamLevelStreamConfig.getGroupId(),
          kafkaStreamLevelStreamConfig.getBootstrapServers());

  synchronized (KafkaStreamLevelConsumerManager.class) {
    // If we have the consumer and it's not already acquired, return it, otherwise error out if it's already acquired
    if (CONSUMER_FOR_CONFIG_KEY.containsKey(configKey)) {
      KafkaConsumer kafkaConsumer = CONSUMER_FOR_CONFIG_KEY.get(configKey);
      if (CONSUMER_RELEASE_TIME.get(kafkaConsumer).equals(IN_USE)) {
        throw new RuntimeException("Consumer " + kafkaConsumer + " already in use!");
      } else {
        LOGGER.info("Reusing kafka consumer with id {}", kafkaConsumer);
        CONSUMER_RELEASE_TIME.put(kafkaConsumer, IN_USE);
        return kafkaConsumer;
      }
    }

    LOGGER.info("Creating new kafka consumer and iterator for topic {}",
        kafkaStreamLevelStreamConfig.getKafkaTopicName());

    // Create the consumer

    Properties consumerProp = new Properties();
    consumerProp.putAll(kafkaStreamLevelStreamConfig.getKafkaConsumerProperties());
    consumerProp.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, kafkaStreamLevelStreamConfig.getBootstrapServers());
    consumerProp.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName());
    consumerProp.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, BytesDeserializer.class.getName());
    if (consumerProp.containsKey(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG) && consumerProp
        .getProperty(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG).equals("smallest")) {
      consumerProp.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
    }
    KafkaConsumer consumer = new KafkaConsumer<>(consumerProp);
    consumer.subscribe(Collections.singletonList(kafkaStreamLevelStreamConfig.getKafkaTopicName()));

    // Mark both the consumer and iterator as acquired
    CONSUMER_FOR_CONFIG_KEY.put(configKey, consumer);
    CONSUMER_RELEASE_TIME.put(consumer, IN_USE);

    LOGGER.info("Created consumer with id {} for topic {}", consumer, kafkaStreamLevelStreamConfig.getKafkaTopicName());

    return consumer;
  }
}