Java Code Examples for org.apache.kafka.clients.consumer.Consumer#seekToBeginning()

The following examples show how to use org.apache.kafka.clients.consumer.Consumer#seekToBeginning() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: KeycloakClientCredentialsWithJwtValidationAuthzTest.java    From strimzi-kafka-oauth with Apache License 2.0 6 votes vote down vote up 
static void consumeFail(Consumer<String, String> consumer, String topic) {
    TopicPartition partition = new TopicPartition(topic, 0);
    consumer.assign(Arrays.asList(partition));

    try {
        while (consumer.partitionsFor(topic, Duration.ofSeconds(1)).size() == 0) {
            System.out.println("No assignment yet for consumer");
        }

        consumer.seekToBeginning(Arrays.asList(partition));
        consumer.poll(Duration.ofSeconds(1));

        Assert.fail("Should fail with TopicAuthorizationException");
    } catch (TopicAuthorizationException e) {
    }
}
Example 2
Source File: AbstractKafkaBasedConnectorTask.java    From brooklin with BSD 2-Clause "Simplified" License 6 votes vote down vote up 
private void seekToStartPosition(Consumer<?, ?> consumer, Set<TopicPartition> partitions, String strategy) {
  // We would like to consume from latest when there is no offset. However, if we rewind due to exception, we want
  // to rewind to earliest to make sure the messages which are added after we start consuming don't get skipped
  if (_startOffsets.isPresent()) {
    _logger.info("Datastream is configured with StartPosition. Trying to start from {}", _startOffsets.get());
    seekToOffset(consumer, partitions, _startOffsets.get());
  } else {
    if (CONSUMER_AUTO_OFFSET_RESET_CONFIG_LATEST.equals(strategy)) {
      _logger.info("Datastream was not configured with StartPosition. Seeking to end for partitions: {}", partitions);
      consumer.seekToEnd(partitions);
    } else {
      _logger.info("Datastream was not configured with StartPosition. Seeking to beginning for partitions: {}",
          partitions);
      consumer.seekToBeginning(partitions);
    }
  }
}
Example 3
Source File: Seek.java    From apicurio-registry with Apache License 2.0 5 votes vote down vote up 
@Override
public void accept(Consumer<?, ?> consumer, TopicPartition topicPartition) {
    Long absOffset = null;
    switch (seek()) {
        case FROM_BEGINNING:
            consumer.seekToBeginning(Collections.singletonList(topicPartition));
            log.info("seekToBeginning: {}", topicPartition);
            if (offset != 0L) {
                absOffset = Math.max(0L, consumer.position(topicPartition) + offset);
            }
            break;
        case FROM_END:
            consumer.seekToEnd(Collections.singletonList(topicPartition));
            log.info("seekToEnd: {}", topicPartition);
            if (offset != 0L) {
                absOffset = Math.max(0L, consumer.position(topicPartition) + offset);
            }
            break;
        case FROM_CURRENT:
            if (offset != 0L) {
                absOffset = Math.max(0L, consumer.position(topicPartition) + offset);
            }
            break;
        case TO_ABSOLUTE:
            absOffset = offset;
            break;
    }
    if (absOffset != null) {
        log.info("seek: {} to offset: {}", topicPartition, absOffset);
        consumer.seek(topicPartition, absOffset);
    }
}
Example 4
Source File: KeycloakAccessTokenWithIntrospectValidationTest.java    From strimzi-kafka-oauth with Apache License 2.0 5 votes vote down vote up 
@Test
public void doTest() throws Exception {
    System.out.println("==== KeycloakAccessTokenWithIntrospectValidationTest ====");

    final String topic = "KeycloakAccessTokenWithIntrospectValidationTest";
    final String tokenEndpointUri = "http://" + HOST + ":8080/auth/realms/" + REALM + "/protocol/openid-connect/token";

    // first, request access token using client id and secret
    TokenInfo info = OAuthAuthenticator.loginWithClientSecret(URI.create(tokenEndpointUri), null, null, CLIENT_ID, CLIENT_SECRET, true, null, null);


    Properties defaults = new Properties();
    defaults.setProperty(ClientConfig.OAUTH_TOKEN_ENDPOINT_URI, tokenEndpointUri);
    defaults.setProperty(ClientConfig.OAUTH_ACCESS_TOKEN, info.token());
    defaults.setProperty(ClientConfig.OAUTH_USERNAME_CLAIM, "preferred_username");

    ConfigProperties.resolveAndExportToSystemProperties(defaults);

    Properties producerProps = buildProducerConfig();
    Producer<String, String> producer = new KafkaProducer<>(producerProps);

    producer.send(new ProducerRecord<>(topic, "The Message")).get();
    System.out.println("Produced The Message");

    Properties consumerProps = buildConsumerConfig();
    Consumer<String, String> consumer = new KafkaConsumer<>(consumerProps);

    TopicPartition partition = new TopicPartition(topic, 0);
    consumer.assign(Arrays.asList(partition));

    while (consumer.partitionsFor(topic, Duration.ofSeconds(1)).size() == 0) {
        System.out.println("No assignment yet for consumer");
    }
    consumer.seekToBeginning(Arrays.asList(partition));

    ConsumerRecords<String, String> records = consumer.poll(Duration.ofSeconds(1));

    Assert.assertEquals("Got message", 1, records.count());
    Assert.assertEquals("Is message text: 'The Message'", "The Message", records.iterator().next().value());
}
Example 5
Source File: KeycloakClientCredentialsWithJwtValidationAuthzTest.java    From strimzi-kafka-oauth with Apache License 2.0 5 votes vote down vote up 
static void consume(Consumer<String, String> consumer, String topic) {
    TopicPartition partition = new TopicPartition(topic, 0);
    consumer.assign(Arrays.asList(partition));

    while (consumer.partitionsFor(topic, Duration.ofSeconds(1)).size() == 0) {
        System.out.println("No assignment yet for consumer");
    }

    consumer.seekToBeginning(Arrays.asList(partition));
    ConsumerRecords<String, String> records = consumer.poll(Duration.ofSeconds(10));

    Assert.assertTrue("Got message", records.count() >= 1);
}
Example 6
Source File: KeycloakRefreshTokenWithJwtValidationTest.java    From strimzi-kafka-oauth with Apache License 2.0 5 votes vote down vote up 
@Test
public void doTest() throws Exception {
    System.out.println("==== KeycloakRefreshTokenWithJwtValidationTest ====");

    final String topic = "KeycloakRefreshTokenWithJwtValidationTest";
    final String tokenEndpointUri = "http://" + HOST + ":8080/auth/realms/" + REALM + "/protocol/openid-connect/token";

    String refreshToken = loginWithClientSecretForRefreshToken(URI.create(tokenEndpointUri), CLIENT_ID, CLIENT_SECRET);

    Properties defaults = new Properties();
    defaults.setProperty(ClientConfig.OAUTH_TOKEN_ENDPOINT_URI, tokenEndpointUri);
    defaults.setProperty(ClientConfig.OAUTH_CLIENT_ID, CLIENT_ID);
    defaults.setProperty(ClientConfig.OAUTH_CLIENT_SECRET, CLIENT_SECRET);
    defaults.setProperty(ClientConfig.OAUTH_REFRESH_TOKEN, refreshToken);
    defaults.setProperty(ClientConfig.OAUTH_USERNAME_CLAIM, "preferred_username");

    ConfigProperties.resolveAndExportToSystemProperties(defaults);

    Properties producerProps = buildProducerConfig();
    Producer<String, String> producer = new KafkaProducer<>(producerProps);

    producer.send(new ProducerRecord<>(topic, "The Message")).get();
    System.out.println("Produced The Message");

    Properties consumerProps = buildConsumerConfig();
    Consumer<String, String> consumer = new KafkaConsumer<>(consumerProps);

    TopicPartition partition = new TopicPartition(topic, 0);
    consumer.assign(Arrays.asList(partition));

    while (consumer.partitionsFor(topic, Duration.ofSeconds(1)).size() == 0) {
        System.out.println("No assignment yet for consumer");
    }
    consumer.seekToBeginning(Arrays.asList(partition));

    ConsumerRecords<String, String> records = consumer.poll(Duration.ofSeconds(1));

    Assert.assertEquals("Got message", 1, records.count());
    Assert.assertEquals("Is message text: 'The Message'", "The Message", records.iterator().next().value());
}
Example 7
Source File: HydraClientCredentialsWithJwtValidationTest.java    From strimzi-kafka-oauth with Apache License 2.0 4 votes vote down vote up 
@Test
public void doTest() throws Exception {
    System.out.println("==== HydraClientCredentialsWithJwtValidationTest ====");

    final String topic = "HydraClientCredentialsWithJwtValidationTest";
    final String tokenEndpointUri = "https://" + HOST + ":4444/oauth2/token";

    Properties defaults = new Properties();
    defaults.setProperty(ClientConfig.OAUTH_TOKEN_ENDPOINT_URI, tokenEndpointUri);
    defaults.setProperty(ClientConfig.OAUTH_CLIENT_ID, "kafka-producer-client");
    defaults.setProperty(ClientConfig.OAUTH_CLIENT_SECRET, "kafka-producer-client-secret");
    defaults.setProperty(ClientConfig.OAUTH_TOKEN_ENDPOINT_URI, tokenEndpointUri);

    defaults.setProperty(ClientConfig.OAUTH_SSL_TRUSTSTORE_LOCATION, "../docker/target/kafka/certs/ca-truststore.p12");
    defaults.setProperty(ClientConfig.OAUTH_SSL_TRUSTSTORE_PASSWORD, "changeit");
    defaults.setProperty(ClientConfig.OAUTH_SSL_TRUSTSTORE_TYPE, "pkcs12");

    ConfigProperties.resolveAndExportToSystemProperties(defaults);

    Properties producerProps = buildProducerConfig();
    Producer<String, String> producer = new KafkaProducer<>(producerProps);

    producer.send(new ProducerRecord<>(topic, "The Message")).get();
    System.out.println("Produced The Message");

    Properties consumerProps = buildConsumerConfig();
    Consumer<String, String> consumer = new KafkaConsumer<>(consumerProps);

    TopicPartition partition = new TopicPartition(topic, 0);
    consumer.assign(Arrays.asList(partition));

    while (consumer.partitionsFor(topic, Duration.ofSeconds(1)).size() == 0) {
        System.out.println("No assignment yet for consumer");
    }
    consumer.seekToBeginning(Arrays.asList(partition));

    ConsumerRecords<String, String> records = consumer.poll(Duration.ofSeconds(1));

    Assert.assertEquals("Got message", 1, records.count());
    Assert.assertEquals("Is message text: 'The Message'", "The Message", records.iterator().next().value());
}
Example 8
Source File: HydraOpaqueAccessTokenWithIntrospectValidationTest.java    From strimzi-kafka-oauth with Apache License 2.0 4 votes vote down vote up 
@Test
public void doTest() throws Exception {
    System.out.println("==== HydraOpaqueAccessTokenWithIntrospectValidationTest ====");

    final String topic = "HydraOpaqueAccessTokenWithIntrospectValidationTest";
    final String tokenEndpointUri = "https://" + HOST + ":4444/oauth2/token";

    Properties defaults = new Properties();
    defaults.setProperty(ClientConfig.OAUTH_TOKEN_ENDPOINT_URI, tokenEndpointUri);
    defaults.setProperty(ClientConfig.OAUTH_ACCESS_TOKEN_IS_JWT, "false");

    defaults.setProperty(ClientConfig.OAUTH_SSL_TRUSTSTORE_LOCATION, "../docker/target/kafka/certs/ca-truststore.p12");
    defaults.setProperty(ClientConfig.OAUTH_SSL_TRUSTSTORE_PASSWORD, "changeit");
    defaults.setProperty(ClientConfig.OAUTH_SSL_TRUSTSTORE_TYPE, "pkcs12");

    ConfigProperties.resolveAndExportToSystemProperties(defaults);

    // Request access token using client id and secret, and trustore configuration
    TokenInfo info = OAuthAuthenticator.loginWithClientSecret(URI.create(tokenEndpointUri),
            ConfigUtil.createSSLFactory(new ClientConfig()),
            null, CLIENT_ID, CLIENT_SECRET, true, null, null);

    // Configure received token for Kafka client auth
    defaults.setProperty(ClientConfig.OAUTH_ACCESS_TOKEN, info.token());
    ConfigProperties.resolveAndExportToSystemProperties(defaults);


    Properties producerProps = buildProducerConfig();
    Producer<String, String> producer = new KafkaProducer<>(producerProps);

    producer.send(new ProducerRecord<>(topic, "The Message")).get();
    System.out.println("Produced The Message");

    Properties consumerProps = buildConsumerConfig();
    Consumer<String, String> consumer = new KafkaConsumer<>(consumerProps);

    TopicPartition partition = new TopicPartition(topic, 0);
    consumer.assign(Arrays.asList(partition));

    while (consumer.partitionsFor(topic, Duration.ofSeconds(1)).size() == 0) {
        System.out.println("No assignment yet for consumer");
    }
    consumer.seekToBeginning(Arrays.asList(partition));

    ConsumerRecords<String, String> records = consumer.poll(Duration.ofSeconds(1));

    Assert.assertEquals("Got message", 1, records.count());
    Assert.assertEquals("Is message text: 'The Message'", "The Message", records.iterator().next().value());
}
Example 9
Source File: KeycloakClientCredentialsWithJwtValidationTest.java    From strimzi-kafka-oauth with Apache License 2.0 4 votes vote down vote up 
@Test
public void doTest() throws Exception {
    System.out.println("==== KeycloakClientCredentialsWithJwtValidationTest + test EC ====");

    Properties p = System.getProperties();
    for (Object key: p.keySet()) {
        System.out.println("" + key + "=" + p.get(key));
    }

    final String topic = "KeycloakClientCredentialsWithJwtValidationTest";
    final String tokenEndpointUri = "http://" + HOST + ":8080/auth/realms/" + REALM + "/protocol/openid-connect/token";

    Properties defaults = new Properties();
    defaults.setProperty(ClientConfig.OAUTH_TOKEN_ENDPOINT_URI, tokenEndpointUri);
    defaults.setProperty(ClientConfig.OAUTH_CLIENT_ID, "kafka-producer-client");
    defaults.setProperty(ClientConfig.OAUTH_CLIENT_SECRET, "kafka-producer-client-secret");
    defaults.setProperty(ClientConfig.OAUTH_USERNAME_CLAIM, "preferred_username");

    ConfigProperties.resolveAndExportToSystemProperties(defaults);

    Properties producerProps = buildProducerConfig();
    Producer<String, String> producer = new KafkaProducer<>(producerProps);

    producer.send(new ProducerRecord<>(topic, "The Message")).get();
    System.out.println("Produced The Message");

    Properties consumerProps = buildConsumerConfig();
    Consumer<String, String> consumer = new KafkaConsumer<>(consumerProps);

    TopicPartition partition = new TopicPartition(topic, 0);
    consumer.assign(Arrays.asList(partition));

    while (consumer.partitionsFor(topic, Duration.ofSeconds(1)).size() == 0) {
        System.out.println("No assignment yet for consumer");
    }
    consumer.seekToBeginning(Arrays.asList(partition));

    ConsumerRecords<String, String> records = consumer.poll(Duration.ofSeconds(1));

    Assert.assertEquals("Got message", 1, records.count());
    Assert.assertEquals("Is message text: 'The Message'", "The Message", records.iterator().next().value());
}