Python kafka.errors.KafkaError() Examples

def send(self, logentry):
        try:
            # send() has a (max_block_ms) timeout and get() has a (max_block_ms) timeout
            # for an upper bound of 2x(max_block_ms) before guaranteed delivery
            future = self._producer.send(
                self.topic, logentry.to_dict(), timestamp_ms=epoch_ms(logentry.datetime)
            )
            record_metadata = future.get(timeout=self.max_block_ms)
            assert future.succeeded
        except KafkaTimeoutError as kte:
            logger.exception("KafkaLogsProducer timeout sending log to Kafka: %s", kte)
            raise LogSendException("KafkaLogsProducer timeout sending log to Kafka: %s" % kte)
        except KafkaError as ke:
            logger.exception("KafkaLogsProducer error sending log to Kafka: %s", ke)
            raise LogSendException("KafkaLogsProducer error sending log to Kafka: %s" % ke)
        except Exception as e:
            logger.exception("KafkaLogsProducer exception sending log to Kafka: %s", e)
            raise LogSendException("KafkaLogsProducer exception sending log to Kafka: %s" % e) 

def commit_offsets(
        self, internal_name: Tuple[str, str], content_type: str, request_data: dict, cluster_metadata: dict
    ):
        self.log.info("Committing offsets for %s", internal_name)
        self._assert_consumer_exists(internal_name, content_type)
        if request_data:
            self._assert_has_key(request_data, "offsets", content_type)
        payload = {}
        for el in request_data.get("offsets", []):
            for k in ["partition", "offset"]:
                convert_to_int(el, k, content_type)
            # If we commit for a partition that does not belong to this consumer, then the internal error raised
            # is marked as retriable, and thus the commit method will remain blocked in what looks like an infinite loop
            self._topic_and_partition_valid(cluster_metadata, el, content_type)
            payload[TopicPartition(el["topic"], el["partition"])] = OffsetAndMetadata(el["offset"] + 1, None)

        async with self.consumer_locks[internal_name]:
            consumer = self.consumers[internal_name].consumer
            payload = payload or None
            try:
                consumer.commit(offsets=payload)
            except KafkaError as e:
                KarapaceBase.internal_error(message=f"error sending commit request: {e}", content_type=content_type)
        empty_response() 

def create_partitions_in_topic(self, partitions, **kwargs):
        """
        create partitions in topic
        Arguments:
          partitions(list) : list of ['topic_name','num_partitions'] lists
          example : [['topic1',4], ['topic2',5]]
          timeout(int): timeout in milliseconds
        Returns:
          result(bool) : False if exception occures, True otherwise
        """
        timeout = kwargs.get("timeout", None)
        validate = kwargs.get("validate", False)
        topic_partitions = {tup[0]:NewPartitions(total_count=tup[1]) for tup in partitions}
        print_info("creating partitions in topic")
        try:
            self.kafka_client.create_partitions(topic_partitions=topic_partitions,
                                                timeout_ms=timeout,
                                                validate_only=validate)
            result = True
        except KafkaError as exc:
            print_error("Exception during creating partitions - {}".format(exc))
            result = False
        return result 

def subscribe_to_topics(self, topics, **kwargs):
        """
        Subscribe to list of specified topics.
        Arguments:
          topics(list): list of topic names to subscribe
          pattern(list): list of topic name patterns to subscribe
          listener(func): callback function
        Returns:
          result(bool) : False if exception occures, True otherwise
        """
        pattern = kwargs.get("pattern", None)
        listener = kwargs.get("listener", None)
        print_info("subscribe to topics {}".format(topics))
        try:
            self.kafka_consumer.subscribe(topics=topics,
                                          pattern=pattern,
                                          listener=listener)
            result = True
        except KafkaError as exc:
            print_error("Exception during subscribing to topics - {}".format(exc))
            result = False
        return result 

def get_topics(self):
        """
        Get subscribed topics of the consumer.
        Arguments:
          None.
        Returns:
          topic_list(list of lists): list of [topic, partition] lists
            example : [[topic1,1], [topic2,2]]
        """
        print_info("get all the topics consumer is subscribed to")
        try:
            topic_partitions = self.kafka_consumer.assignment()
            topic_list = [[topic_partition.topic, topic_partition.partition] \
                       for topic_partition in topic_partitions]
        except KafkaError as exc:
            print_error("Exception during getting assigned partitions - {}".format(exc))
            topic_list = None
        return topic_list 

def assign_partitions(self, partitions):
        """
        Assign partitions to consumer.
        Arguments:
          partitions(list) : list of [topic, partition] lists
            example : [[topic1,1], [topic2,1]]
        Returns:
            None.
        """
        print_info("assigning partitions to consumer {}".format(partitions))
        topic_partitions = [TopicPartition(topic=tup[0], partition=tup[1]) for tup in partitions]
        try:
            self.kafka_consumer.assign(topic_partitions)
            result = True
        except KafkaError as exc:
            print_error("Exception during assiging partitions - {}".format(exc))
            result = False
        return result 

def seek_to_position(self, topic, partition, offset):
        """
        Seek to the given offset.
        Arguments:
          topic(str): topic name
          partition(int): partition number
          offset(int): offset number
        Returns:
          result(bool) : False if exception occures, True otherwise
        """
        print_info("seeking to position {}:{}:{}".format(topic, partition, offset))
        topic_partition = TopicPartition(topic=topic, partition=partition)
        try:
            self.kafka_consumer.seek(partition=topic_partition, offset=offset)
            result = True
        except KafkaError as exc:
            print_error("Exception during seek - {}".format(exc))
            result = False
        return result 

def unsubscribe_to_topics(self):
        """
        Unsubscribe to all topics.
        Arguments: None.
        Returns:
          result(bool) : False if exception occures, True otherwise
        """
        print_info("unsubscribe to all topics")
        try:
            self.kafka_consumer.unsubscribe()
            result = True
        except KafkaError as exc:
            print_error("Exception during unsubscibing to topics - {}".format(exc))
            result = False
        return result 

def test_coordinator__send_req(self):
        client = AIOKafkaClient(loop=self.loop, bootstrap_servers=self.hosts)
        await client.bootstrap()
        self.add_cleanup(client.close)
        subscription = SubscriptionState(loop=self.loop)
        subscription.subscribe(topics=set(['topic1']))
        coordinator = GroupCoordinator(
            client, subscription, loop=self.loop,
            group_id='test-my-group', session_timeout_ms=6000,
            heartbeat_interval_ms=1000)
        self.add_cleanup(coordinator.close)

        request = OffsetCommitRequest[2](topics=[])

        # We did not call ensure_coordinator_known yet
        with self.assertRaises(Errors.GroupCoordinatorNotAvailableError):
            await coordinator._send_req(request)

        await coordinator.ensure_coordinator_known()
        self.assertIsNotNone(coordinator.coordinator_id)

        with mock.patch.object(client, "send") as mocked:
            async def mock_send(*args, **kw):
                raise Errors.KafkaError("Some unexpected error")
            mocked.side_effect = mock_send

            # _send_req should mark coordinator dead on errors
            with self.assertRaises(Errors.KafkaError):
                await coordinator._send_req(request)
            self.assertIsNone(coordinator.coordinator_id) 

def create_topics(self, topic_sets, **kwargs):
        """
        create topics for the producer or consumer to use
        Arguments:
         topic_sets(list) : list of
         ['topic_name', 'num_partitions', 'replication_factor'] lists
         example : ['topic1',1,1]
         timeout(int): time in milliseconds
        Returns:
          result(bool) : False if exception occures, True otherwise
         None.
        """
        timeout = kwargs.get("timeout", None)
        validate = kwargs.get("validate", False)
        new_topics = [NewTopic(name=tup[0], num_partitions=tup[1],\
                      replication_factor=tup[2]) for tup in topic_sets]
        print_info("creating topics")
        try:
            self.kafka_client.create_topics(new_topics=new_topics,
                                            timeout_ms=timeout,
                                            validate_only=validate)
            result = True
        except KafkaError as exc:
            print_error("Exception during creating topics - {}".format(exc))
            result = False
        return result 

def __init__(self, **configs):
        """
        create a kafka client
        """
        print_info("Creating kafka client")
        try:
            self.kafka_client = KafkaAdminClient(**configs)
        except KafkaError as exc:
            print_error("kafka client - Exception during connecting to broker- {}".format(exc)) 

def send_messages(self, topic, value=None, **kwargs):
        """
        Publish messages to the desired topic
        Arguments:
          topic(str): topic name to publish messages
          partition(int): partition nubmer
          key(str): key name
          value(str): message to publish
        Returns:
          result(bool) : False if exception occures, True otherwise
        """
        partition = kwargs.get("partition", None)
        headers = kwargs.get("headers", None)
        timestamp = kwargs.get("timestamp", None)
        key = kwargs.get("key", None)
        print_info("publishing messages to the topic")
        try:
            self.kafka_producer.send(topic=topic,
                                     value=value,
                                     partition=partition,
                                     key=key,
                                     headers=headers,
                                     timestamp_ms=timestamp)
            self.kafka_producer.flush()
            result = True
        except KafkaError as exc:
            print_error("Exception during publishing messages - {}".format(exc))
            result = False
        return result 

def __init__(self, **configs):
        """
        Create kafka producer object
        """
        print_info("Creating kafka producer")
        try:
            self.kafka_producer = KafkaProducer(**configs)
        except KafkaError as exc:
            print_error("kafka producer - Exception during connecting to broker - {}".format(exc)) 

def listen_for_messages(msg, consumer, application_source_id):  # noqa: C901
    """
    Listen for Platform-Sources kafka messages.

    Args:
        consumer (Consumer): Kafka consumer object
        application_source_id (Integer): Cost Management's current Application Source ID. Used for
            kafka message filtering.

    Returns:
        None

    """
    try:
        try:
            msg = get_sources_msg_data(msg, application_source_id)
            offset = msg.get("offset")
            partition = msg.get("partition")
        except SourcesMessageError:
            return
        if msg:
            LOG.info(f"Processing message offset: {offset} partition: {partition}")
            topic_partition = TopicPartition(topic=Config.SOURCES_TOPIC, partition=partition, offset=offset)
            LOG.info(f"Cost Management Message to process: {str(msg)}")
            try:
                with transaction.atomic():
                    process_message(application_source_id, msg)
                    consumer.commit()
            except (IntegrityError, InterfaceError, OperationalError) as err:
                connection.close()
                LOG.error(f"{type(err).__name__}: {err}")
                rewind_consumer_to_retry(consumer, topic_partition)
            except SourcesHTTPClientError as err:
                LOG.error(err)
                rewind_consumer_to_retry(consumer, topic_partition)
            except SourceNotFoundError:
                LOG.warning(f"Source not found in platform sources. Skipping msg: {msg}")
                consumer.commit()

    except KafkaError as error:
        LOG.error(f"[listen_for_messages] Kafka error encountered: {type(error).__name__}: {error}", exc_info=True)
    except Exception as error:
        LOG.error(f"[listen_for_messages] UNKNOWN error encountered: {type(error).__name__}: {error}", exc_info=True) 

def __init__(self, *topics, **configs):
        """
        Create Kafka Consumer object
        """
        print_info("creating kafka consumer")
        try:
            self.kafka_consumer = KafkaConsumer(*topics, **configs)
        except KafkaError as exc:
            print_error("Kafka consumer - Exception during connecting to broker - {}".format(exc)) 

def pop_queries_for_worker(self, worker_id: str, batch_size: int) -> List[Query]:
        name = f'workers_{worker_id}_queries'

        query_consumer = KafkaConsumer(name, bootstrap_servers=self.connection_url, auto_offset_reset='earliest', group_id=QUERIES_QUEUE)
        
        partition = TopicPartition(name, 0)
        partitiondic = query_consumer.end_offsets([partition])
        offsetend = partitiondic.get(partition, None)
        if offsetend == 0:
            query_consumer.close()
            return []
        try:
            queries = []
            while True:
                record = next(query_consumer)
                queries.append(record.value)
                query_consumer.commit()
                if record.offset >= offsetend-1 or len(queries) == batch_size:
                    break
                
            queries = [pickle.loads(x) for x in queries]
            query_consumer.close()
            return queries
        except KafkaError:
            query_consumer.close()
            return [] 

def take_prediction_for_worker(self, worker_id: str, query_id: str) -> Union[Prediction, None]:
        name = f'workers_{worker_id}_{query_id}_prediction'

        prediction_consumer = KafkaConsumer(name, bootstrap_servers=self.connection_url, auto_offset_reset='earliest', group_id=PREDICTIONS_QUEUE)
        prediction = None
        try:
            prediction = next(prediction_consumer).value
            prediction_consumer.commit()
            prediction = pickle.loads(prediction)
        except KafkaError:
            pass
        prediction_consumer.close()
        logger.info(f'Took prediction for query "{query_id}" from worker "{worker_id}"')
        return prediction 

def send_json_data(self, params):
        try:
            parmas_message = json.dumps(params)
            producer = self.producer
            producer.send(self.kafkatopic, parmas_message.encode('utf-8'))
            producer.flush()
        except KafkaError as e:
            print(e) 

def test_kafka_connection_metrics_listen_for_messages(self, mock_start, mock_sleep):
        """Test check_kafka_connection increments kafka connection errors on KafkaError."""
        connection_errors_before = WORKER_REGISTRY.get_sample_value("kafka_connection_errors_total")
        source_integration.is_kafka_connected()
        connection_errors_after = WORKER_REGISTRY.get_sample_value("kafka_connection_errors_total")
        self.assertEqual(connection_errors_after - connection_errors_before, 1) 

def getone(self):
        for msg in self.preloaded_messages:
            return msg
        raise KafkaError("Closing Mock Consumer") 

def seek(self, topic_partition):
        # This isn't realistic... But it's one way to stop the consumer for our needs.
        raise KafkaError("Seek to commited. Closing...")