Jave send to kafka

@Test
public void sendToKafkaTest() throws IOException, ClassHierarchyException, CallGraphBuilderCancelException {
    KafkaProducer<Object, String> producer = Mockito.mock(KafkaProducer.class);

    Mockito.when(producer.send(Mockito.any())).thenReturn(null);

    JSONObject coordinateJSON = new JSONObject("{\n" +
            "    \"groupId\": \"org.slf4j\",\n" +
            "    \"artifactId\": \"slf4j-api\",\n" +
            "    \"version\": \"1.7.29\",\n" +
            "    \"date\":\"1574072773\"\n" +
            "}");

    walaPlugin.consume(coordinateJSON.toString());

    var coordinate = new MavenCoordinate("org.slf4j", "slf4j-api", "1.7.29");
    var revisionCallGraph = PartialCallGraph.createExtendedRevisionCallGraph(coordinate,
            1574072773);

    assertTrue(walaPlugin.produce().isPresent());
    assertEquals(revisionCallGraph.toJSON().toString(), walaPlugin.produce().get());
}
 

public void sendMessageToKafka(String resultTopic, String key, DbusMessage dbusMessage,
                               AtomicLong sendCnt, AtomicLong recvCnt, AtomicBoolean isError) throws Exception {
    if (stringProducer == null) {
        throw new Exception("producer is null, can't send to kafka!");
    }

    ProducerRecord record = new ProducerRecord<>(resultTopic, key, dbusMessage.toString());
    sendCnt.getAndIncrement();
    stringProducer.send(record, (metadata, e) -> {
        if (e != null) {
            logger.error("Send Message to kafka exception!", e);
            isError.set(true);
        } else {
            recvCnt.getAndIncrement();
        }
    });
}
 

@Test
public void shouldSendRequestToServer() throws Exception {
    CompletableFuture<Request> future = new CompletableFuture<>();
    RequestHandler handler = request -> {
        future.complete(request);
        return Response.newBuilder()
                .withBody(new TestResponseBody("Response"))
                .withCorrelationId(request.getCorrelationId())
                .withLast(true)
                .buildSuccess();
    };
    handlerWrapper.setDelegate(handler);

    Request request = Request.newBuilder()
            .withBody(new TestRequestBody("RequestResponseTest"))
            .withSingleReply(true)
            .build();

    client.push(request);

    Request receivedRequest = future.get(10, TimeUnit.SECONDS);
    assertEquals(request, receivedRequest);
}
 

@Test
public void sendMessageToKafkaVerifyStatsDExecutorCalled() {
    KafkaMessageProcessor kafkaMessageProcessorSpy = spy(kafkaMessageProcessor);

    when(kafkaMessageProcessorSpy.getTopic()).thenReturn("default-topic");
    when(kafkaMessageProcessorSpy.getSerializerType()).thenReturn(MessageSerializerType.STRING_SERIALIZER);

    JsonObject jsonObjectMock = mock(JsonObject.class);

    when(event.body()).thenReturn(jsonObjectMock);
    when(jsonObjectMock.getString(EventProperties.TOPIC)).thenReturn("");
    when(jsonObjectMock.getString(EventProperties.PAYLOAD)).thenReturn("test payload");

    StringMessageHandler messageHandler = mock(StringMessageHandler.class);
    when(messageHandlerFactory.createMessageHandler(any(MessageSerializerType.class))).
            thenReturn(messageHandler);

    kafkaMessageProcessorSpy.sendMessageToKafka(producer, event);

    verify(statsDClient, times(1)).recordExecutionTime(anyString(), anyLong());
}
 

@Test
public void sendMessageToKafka() {
    KafkaMessageProcessor kafkaMessageProcessorSpy = spy(kafkaMessageProcessor);

    when(kafkaMessageProcessorSpy.getTopic()).thenReturn("default-topic");
    when(kafkaMessageProcessorSpy.getSerializerType()).thenReturn(MessageSerializerType.STRING_SERIALIZER);

    JsonObject jsonObjectMock = mock(JsonObject.class);

    when(event.body()).thenReturn(jsonObjectMock);
    when(jsonObjectMock.getString(EventProperties.TOPIC)).thenReturn("");
    when(jsonObjectMock.getString(EventProperties.PAYLOAD)).thenReturn("test payload");
    when(jsonObjectMock.getString(EventProperties.PART_KEY)).thenReturn("test partition key");

    StringMessageHandler messageHandler = mock(StringMessageHandler.class);
    when(messageHandlerFactory.createMessageHandler(any(MessageSerializerType.class))).
            thenReturn(messageHandler);

    kafkaMessageProcessorSpy.sendMessageToKafka(producer, event);

    verify(messageHandler, times(1)).send(producer, "default-topic", "test partition key", event.body());
}
 

private void sendToKafka(KafkaConfig kafkaConfig, Event event, Callback callback) {
    String value = JSON.toJSONString(event);
    ProducerRecord<String, String> record = new ProducerRecord<>(kafkaConfig.getTopic(), getPrimaryKey(kafkaConfig, event), value);
    String[] errors = new String[1];
    try {
        if (callback == null) {
            producer.send(record);
        } else {
            producer.send(record,
                    (metadata, exception) -> {
                        if (exception != null) {
                            errors[0] = exception.getMessage();
                            KafkaPublisher.this.onFailure(callback, exception);
                        } else {
                            KafkaPublisher.this.onSuccess(callback);
                        }
                    });
        }
    } catch (Exception e) {
        errors[0] = e.getMessage();
        throw new RuntimeException(e);
    } finally {
        Statics statics = Statics.createStatics(
                "AppTunnelService",
                event.getSchema(),
                event.getSlotName(),
                event.getTable(),
                1,
                "kafka",
                errors[0] == null ? "" : errors[0]
        );
        TunnelMonitorFactory.getTunnelMonitor().collect(statics);
    }
}
 

protected Future<?> sendToKafka(KafkaProducer<String, String> producer, String kafkaTopic, String message) {
  return producer.send(new ProducerRecord<>(kafkaTopic, message),
                    (recordMetadata, e) -> {
                      if(e != null) {
                        e.printStackTrace(System.err);
                      }
                      numSent.incrementAndGet();
                    }
            );
}
 

public static ProducerRecord prepareRecordToSend(String topicName, ProducerRecord recordToSend) {

        return new ProducerRecord(topicName,
                recordToSend.partition(),
                recordToSend.timestamp(),
                recordToSend.key(),
                recordToSend.value(),
                recordToSend.headers());
    }
 

public static ProducerRecord<Object, Object> prepareJsonRecordToSend(String topicName, ProducerJsonRecord<?> recordToSend) {
    return ProducerRecordBuilder.from(topicName,
            recordToSend.getKey(),
            // --------------------------------------------
            // It's a JSON as String. Nothing to worry !
            // Kafka StringSerializer needs in this format.
            // --------------------------------------------
            recordToSend.getValue().toString())
            .withHeaders(recordToSend.getHeaders())
            .build();
}
 

private Producer<String, String> getProducer() {
    if (producer == null) {

        producer = (Producer) ActionBaseContext.getInstance().getContext().getBean
                ("producer");
    }
    return producer;
}
 

@Override
public void process(Iterable<StratioStreamingMessage> messages) throws Exception {
    List<KeyedMessage<String, String>> kafkaMessages = new ArrayList<>();
    for (StratioStreamingMessage message : messages) {
        kafkaMessages.add(new KeyedMessage<String, String>(message.getStreamName(), getSerializer().deserialize(
                message)));
    }
    getProducer().send(kafkaMessages);
}
 

@Override
public void sendOffsetsToTransaction(Map<TopicPartition, OffsetAndMetadata> offsets, String consumerGroupId) throws ProducerFencedException {
	synchronized (producerClosingLock) {
		ensureNotClosed();
		kafkaProducer.sendOffsetsToTransaction(offsets, consumerGroupId);
	}
}
 

public byte[] getSequencedBinaryPayloadToSend(byte[] payload, KafkaSinkState kafkaSinkState) {
    StringBuilder strPayload = new StringBuilder();
    strPayload.append(sequenceId).append(SEQ_NO_HEADER_FIELD_SEPERATOR)
            .append(kafkaSinkState.lastSentSequenceNo.get())
            .append(SEQ_NO_HEADER_DELIMITER);
    int headerSize = strPayload.toString().length();
    int bufferSize = headerSize + 4 + payload.length;
    ByteBuffer byteBuffer = ByteBuffer.wrap(new byte[bufferSize]);
    byteBuffer.putInt(headerSize);
    byteBuffer.put(strPayload.toString().getBytes(Charset.defaultCharset()));
    byteBuffer.put(payload);
    return byteBuffer.array();
}
 

public void periodicallySendMessageToKafka() {
    AtomicInteger counter = new AtomicInteger();
    Executors.newSingleThreadScheduledExecutor()
            .scheduleAtFixedRate(() -> {
                emitter.send("Hello " + counter.getAndIncrement());
            },
                    1, 1, TimeUnit.SECONDS);
}
 

@Override
public void sendOffsetsToTransaction(Map<TopicPartition, OffsetAndMetadata> offsets, String consumerGroupId) throws ProducerFencedException {
	synchronized (producerClosingLock) {
		ensureNotClosed();
		kafkaProducer.sendOffsetsToTransaction(offsets, consumerGroupId);
	}
}
 

@VisibleForTesting
void sendInternalToProducer(Producer p, NotificationType notificationType, List<String> messages) throws NotificationException {
    String               topic           = PRODUCER_TOPIC_MAP.get(notificationType);
    List<MessageContext> messageContexts = new ArrayList<>();

    for (String message : messages) {
        ProducerRecord record = new ProducerRecord(topic, message);

        if (LOG.isDebugEnabled()) {
            LOG.debug("Sending message for topic {}: {}", topic, message);
        }

        Future future = p.send(record);

        messageContexts.add(new MessageContext(future, message));
    }

    List<String> failedMessages       = new ArrayList<>();
    Exception    lastFailureException = null;

    for (MessageContext context : messageContexts) {
        try {
            RecordMetadata response = context.getFuture().get();

            if (LOG.isDebugEnabled()) {
                LOG.debug("Sent message for topic - {}, partition - {}, offset - {}", response.topic(), response.partition(), response.offset());
            }
        } catch (Exception e) {
            lastFailureException = e;

            failedMessages.add(context.getMessage());
        }
    }

    if (lastFailureException != null) {
        throw new NotificationException(lastFailureException, failedMessages);
    }
}
 

@Override
public void sendOffsetsToTransaction(Map<TopicPartition, OffsetAndMetadata> offsets, String consumerGroupId) throws ProducerFencedException {
  verifyOpen();

  delegateLock.readLock().lock();
  try {
    delegate.sendOffsetsToTransaction(offsets, consumerGroupId);
  } finally {
    delegateLock.readLock().unlock();
  }
}
 

@SuppressWarnings("unchecked")
void sendToDlq(ConsumerRecord<?, ?> consumerRecord, Headers headers,
			String dlqName, String group, Throwable throwable, DlqPartitionFunction partitionFunction,
			MessageHeaders messageHeaders, ContainerProperties.AckMode ackMode) {
	K key = (K) consumerRecord.key();
	V value = (V) consumerRecord.value();
	ProducerRecord<K, V> producerRecord = new ProducerRecord<>(dlqName,
			partitionFunction.apply(group, consumerRecord, throwable),
			key, value, headers);

	StringBuilder sb = new StringBuilder().append(" a message with key='")
			.append(toDisplayString(ObjectUtils.nullSafeToString(key), 50))
			.append("'").append(" and payload='")
			.append(toDisplayString(ObjectUtils.nullSafeToString(value), 50))
			.append("'").append(" received from ")
			.append(consumerRecord.partition());
	ListenableFuture<SendResult<K, V>> sentDlq = null;
	try {
		sentDlq = this.kafkaTemplate.send(producerRecord);
		sentDlq.addCallback(new ListenableFutureCallback<SendResult<K, V>>() {

			@Override
			public void onFailure(Throwable ex) {
				KafkaMessageChannelBinder.this.logger
						.error("Error sending to DLQ " + sb.toString(), ex);
			}

			@Override
			public void onSuccess(SendResult<K, V> result) {
				if (KafkaMessageChannelBinder.this.logger.isDebugEnabled()) {
					KafkaMessageChannelBinder.this.logger
							.debug("Sent to DLQ " + sb.toString());
				}
				if (ackMode == ContainerProperties.AckMode.MANUAL || ackMode == ContainerProperties.AckMode.MANUAL_IMMEDIATE) {
					messageHeaders.get(KafkaHeaders.ACKNOWLEDGMENT, Acknowledgment.class).acknowledge();
				}
			}
		});
	}
	catch (Exception ex) {
		if (sentDlq == null) {
			KafkaMessageChannelBinder.this.logger
					.error("Error sending to DLQ " + sb.toString(), ex);
		}
	}

}
 

@Test
public void testSendToTombstone()
		throws Exception {
	SpringApplication app = new SpringApplication(
			WordCountProcessorApplication.class);
	app.setWebApplicationType(WebApplicationType.NONE);

	try (ConfigurableApplicationContext context = app.run("--server.port=0",
			"--spring.jmx.enabled=false",
			"--spring.cloud.stream.bindings.input.destination=words-1",
			"--spring.cloud.stream.bindings.output.destination=counts-1",
			"--spring.cloud.stream.kafka.streams.bindings.input.consumer.application-id=testKstreamWordCountWithInputBindingLevelApplicationId",
			"--spring.cloud.stream.kafka.streams.binder.configuration.commit.interval.ms=1000",
			"--spring.cloud.stream.kafka.streams.binder.configuration.default.key.serde"
					+ "=org.apache.kafka.common.serialization.Serdes$StringSerde",
			"--spring.cloud.stream.kafka.streams.binder.configuration.default.value.serde"
					+ "=org.apache.kafka.common.serialization.Serdes$StringSerde",
			"--spring.cloud.stream.kafka.streams.bindings.output.producer.valueSerde=org.springframework.kafka.support.serializer.JsonSerde",
			"--spring.cloud.stream.kafka.streams.timeWindow.length=5000",
			"--spring.cloud.stream.kafka.streams.timeWindow.advanceBy=0",
			"--spring.cloud.stream.bindings.input.consumer.concurrency=2",
			"--spring.cloud.stream.kafka.streams.binder.brokers="
					+ embeddedKafka.getBrokersAsString())) {
		receiveAndValidate("words-1", "counts-1");
		// Assertions on StreamBuilderFactoryBean
		StreamsBuilderFactoryBean streamsBuilderFactoryBean = context
				.getBean("&stream-builder-WordCountProcessorApplication-process", StreamsBuilderFactoryBean.class);
		KafkaStreams kafkaStreams = streamsBuilderFactoryBean.getKafkaStreams();
		assertThat(kafkaStreams).isNotNull();
		// Ensure that concurrency settings are mapped to number of stream task
		// threads in Kafka Streams.
		final Properties streamsConfiguration = streamsBuilderFactoryBean.getStreamsConfiguration();
		final Integer concurrency = (Integer) streamsConfiguration
				.get(StreamsConfig.NUM_STREAM_THREADS_CONFIG);
		assertThat(concurrency).isEqualTo(2);

		sendTombStoneRecordsAndVerifyGracefulHandling();

		CleanupConfig cleanup = TestUtils.getPropertyValue(streamsBuilderFactoryBean,
				"cleanupConfig", CleanupConfig.class);
		assertThat(cleanup.cleanupOnStart()).isTrue();
		assertThat(cleanup.cleanupOnStop()).isFalse();
	}
}
 

@Test
public void shouldSendMultipleResponsesToClient() throws Exception {
    RequestHandler handler = request -> Response.newBuilder()
            .withBody(new TestResponseBody("ResponseFromServer"))
            .withCorrelationId(request.getCorrelationId())
            .withLast(request.isSingleReplyExpected())
            .buildSuccess();
    handlerWrapper.setDelegate(handler);

    Request request = Request.newBuilder()
            .withBody(new TestRequestBody("RequestResponseTest"))
            .withSingleReply(false)
            .build();

    CountDownLatch latch = new CountDownLatch(10);
    List<Response> responses = Collections.synchronizedList(new LinkedList<>());
    Consumer<Response> func = response -> {
        responses.add(response);
        latch.countDown();
    };

    client.call(request, func);

    Executor executor = Executors.newFixedThreadPool(2);
    for (int i = 0; i < 9; i++) {
        final int number = i;
        executor.execute(() -> {
            Response response = Response.newBuilder()
                    .withBody(new TestResponseBody(number + "-response"))
                    .withCorrelationId(request.getCorrelationId())
                    .withLast(false)
                    .buildSuccess();
            server.getDispatcher().send(RESPONSE_TOPIC, response);
        });
    }

    latch.await();
    assertEquals(10, responses.size());

    Set<String> correlationIds = responses.stream()
            .map(Response::getCorrelationId).collect(Collectors.toSet());
    assertEquals(1, correlationIds.size());
    assertTrue(correlationIds.contains(request.getCorrelationId()));

    Set<String> bodies = responses.stream()
            .map(Response::getBody)
            .map(responseBody -> (TestResponseBody) responseBody)
            .map(TestResponseBody::getResponseBody)
            .collect(Collectors.toSet());
    assertEquals(10, bodies.size());
    assertTrue(bodies.contains("ResponseFromServer"));
    for (int i = 0; i < 9; i++) {
        assertTrue(bodies.contains(i + "-response"));
    }
}
 

@Test
public void sendMessageToKafka() {
    kafkaMessageProducer.sendString(new StringKafkaMessage("test payload"), event1 -> {
        verify(producer, times(1)).send(new ProducerRecord("default-topic", null, "test payload"));
    });
}
 

@Test
public void sendMessageToKafkaWithTopic() {
    kafkaMessageProducer.sendString(new StringKafkaMessage("test payload", "some partition"), new KafkaOptions().setTopic("foo-topic"), event1 -> {
        verify(producer, times(1)).send(new ProducerRecord("foo-topic", "some partition", "test payload"));
    });
}
 

@Test
public void sendMessageToKafkaWithPartKey() {
    kafkaMessageProducer.sendString(new StringKafkaMessage("test payload", "some partition"), event1 -> {
        verify(producer, times(1)).send(new ProducerRecord("default-topic", "some partition", "test payload"));
    });
}
 

@LogMessage(level = Logger.Level.ERROR)
@Message(id = 18210, value = "Unable to send a record to Kafka ")
void unableToSendRecord(@Cause Throwable t);
 

/**
 * Function to send some test messages to Kafka.
 * We'll get the Kafka producer object as a parameter to this function.
 * We'll generate some test messages, both simple strings and JSON objects, in a couple of
 * loops inside the function. We'll send these test messages to the topic in Kafka.
 *
 * @param producer The Kafka producer we created in the run() method earlier.
 */
private void sendTestMessagesToKafka(KafkaProducer<String, String> producer) {
    /*
    Creating a loop which iterates 10 times, from 0 to 9, and sending a
    simple message to Kafka.
     */
    for (int index = 0; index < 10; index++) {
        sendKafkaMessage("The index is now: " + index, producer, theTechCheckTopicName);
    }

    /*
    Creating a loop which iterates 10 times, from 0 to 9, and creates an instance of JSONObject
    in each iteration. We'll use this simple JSON object to illustrate how we can send a JSON
    object as a message in Kafka.
     */
    for (int index = 0; index < 10; index++) {

        /*
        We'll create a JSON object which will have a bunch of fields, and another JSON object,
        which will be nested inside the first JSON object. This is just to demonstrate how
        complex objects could be serialized and sent to topics in Kafka.
         */
        JSONObject jsonObject = new JSONObject();
        JSONObject nestedJsonObject = new JSONObject();

        try {
            /*
            Adding some random data into the JSON object.
             */
            jsonObject.put("index", index);
            jsonObject.put("message", "The index is now: " + index);

            /*
            We're adding a field in the nested JSON object.
             */
            nestedJsonObject.put("nestedObjectMessage", "This is a nested JSON object with index: " + index);

            /*
            Adding the nexted JSON object to the main JSON object.
             */
            jsonObject.put("nestedJsonObject", nestedJsonObject);

        } catch (JSONException e) {
            logger.error(e.getMessage());
        }

        /*
        We'll now serialize the JSON object we created above, and send it to the same topic in Kafka,
        using the same function we used earlier.
        You can use any JSON library for this, just make sure it serializes your objects properly.
        A popular alternative to the one I've used is Gson.
         */
        sendKafkaMessage(jsonObject.toString(), producer, theTechCheckTopicName);
    }
}
 

public SendRowToKafkaFunction(Config config) {
  this.brokers = config.getString(BROKERS_CONFIG_NAME);
  this.topic = config.getString(TOPIC_CONFIG_NAME);
  this.serializerType = config.getString(SERIALIZER_TYPE_CONFIG_NAME);
  this.config = config;
}
 

@Override
public void sendOffsetsToTransaction(Map<TopicPartition, OffsetAndMetadata> offsets,
    String consumerGroupId)
    throws ProducerFencedException {
  producer.sendOffsetsToTransaction(offsets, consumerGroupId);
}
 

@Override
public void sendOffsetsToTransaction(Map<TopicPartition, OffsetAndMetadata> map, String s) throws ProducerFencedException {
    throw new UnsupportedOperationException();
}