kafka.message.Message Java Examples

private boolean verifyMessage(String key, String message) {
    long lastMessageOffset = KafkaUtils.getOffset(simpleConsumer, kafkaConfig.topic, 0, OffsetRequest.LatestTime()) - 1;
    ByteBufferMessageSet messageAndOffsets = KafkaUtils.fetchMessages(kafkaConfig, simpleConsumer,
            new Partition(Broker.fromString(broker.getBrokerConnectionString()), 0), lastMessageOffset);
    MessageAndOffset messageAndOffset = messageAndOffsets.iterator().next();
    Message kafkaMessage = messageAndOffset.message();
    ByteBuffer messageKeyBuffer = kafkaMessage.key();
    String keyString = null;
    String messageString = new String(Utils.toByteArray(kafkaMessage.payload()));
    if (messageKeyBuffer != null) {
        keyString = new String(Utils.toByteArray(messageKeyBuffer));
    }
    assertEquals(key, keyString);
    assertEquals(message, messageString);
    return true;
}
 

public void testIt() throws Exception
{
	byte[] data = "somedata".getBytes();
	List<String> l = new ArrayList<>();
	
	MessageHandler m = new RawMessageHandler();
	Message message = createMock(Message.class);
	expect(message.payload()).andReturn(ByteBuffer.wrap(data));
	
	BulkRequestBuilder bulkRequestBuilder = createMock(BulkRequestBuilder.class);
	expect(bulkRequestBuilder.add(aryEq(data), eq(0), eq(data.length), eq(false))).andReturn(null);
	replay(message, bulkRequestBuilder);
	
	m.handle(bulkRequestBuilder, message);
	verify(bulkRequestBuilder, message);
}
 

public void testReadMessage() throws Exception
{
	JsonMessageHandler h = new JsonMessageHandler(null);	
	byte[] json = toJson(rec).getBytes();
	Message message = createMock(Message.class);
	expect(message.payload()).andReturn(ByteBuffer.wrap(json));
	replay(message);
	
	try {
		h.readMessage(message);
	} catch (Exception e) {
		fail("This should not fail");
	}
	
	verify(message);
}
 

/**
 * Implement abstract method of AbstractActiveMQSinglePortInputOperator
 */
@Override
public String getTuple(Message message)
{
  String data = "";
  try {
    ByteBuffer buffer = message.payload();
    byte[] bytes = new byte[buffer.remaining()];
    buffer.get(bytes);
    data = new String(bytes);
    //logger.debug("Consuming {}", data);
  } catch (Exception ex) {
    return data;
  }
  return data;
}
 

public void testGettersFromReadMessageReturnedMap() throws Exception
{
	JsonMessageHandler h = new JsonMessageHandler(null);	
	byte[] json = toJson(rec).getBytes();
	Message message = createMock(Message.class);
	
	expect(message.payload()).andReturn(ByteBuffer.wrap(json));
	replay(message);
	
	try {
		h.readMessage(message);
	} catch (Exception e) {
		fail("This should not fail");
	}
	
	assertEquals(h.getIndex(), rec.get("index"));
	assertEquals(h.getType(), rec.get("type"));
	assertEquals(h.getSource(), rec.get("source"));
	assertEquals(h.getId(), rec.get("id"));
	verify(message);
}
 

@SuppressWarnings("unchecked")
@Before
public void setup() throws Exception {
	mockIt = mock(ConsumerIterator.class);
	mockMessageAndMetadata = mock(MessageAndMetadata.class);
	mockChannelProcessor = mock(ChannelProcessor.class);
	mockBuffer = mock(ByteBuffer.class);
	mockMessage = mock(Message.class);
	mockKafkaSource = new KafkaSource();
	
	when(mockMessage.payload()).thenReturn(mockBuffer);
	when(mockMessageAndMetadata.message()).thenReturn(mockMessage);
	
	Field field = AbstractSource.class.getDeclaredField("channelProcessor");
	field.setAccessible(true);
	field.set(mockKafkaSource, mockChannelProcessor);

	field = KafkaSource.class.getDeclaredField("it");
	field.setAccessible(true);
	field.set(mockKafkaSource, mockIt);
}
 

@SuppressWarnings("unchecked")
public Iterable<List<Object>> generateTuples(Message msg) {
    Iterable<List<Object>> tups = null;
    ByteBuffer payload = msg.payload();
    if (payload == null) {
        return null;
    }
    tups = Arrays.asList(Utils.tuple(Utils.toByteArray(payload)));
    return tups;
}
 

private ByteArrayBasedKafkaRecord getMockMessageAndOffset(ByteBuffer payload) {
  MessageAndOffset mockMessageAndOffset = mock(MessageAndOffset.class);
  Message mockMessage = mock(Message.class);
  when(mockMessage.payload()).thenReturn(payload);
  when(mockMessageAndOffset.message()).thenReturn(mockMessage);
  return new Kafka08ConsumerRecord(mockMessageAndOffset, "test", 0);
}
 

@Override
public ByteBufferMessageSet messageSet(String topic, int partition) {
  if (errorMap.containsKey(new TopicAndPartition(topic, partition))) {
    throw new IllegalArgumentException();
  } else {
    // TODO Maybe generate dummy messages here?
    return new ByteBufferMessageSet(Collections.<Message>emptyList());
  }
}
 

/**
 * Fetches the next Kafka message and stuffs the results into the key and
 * value
 *
 * @param key
 * @param payload
 * @param pKey
 * @return true if there exists more events
 * @throws IOException
 */
public boolean getNext(KafkaKey key, BytesWritable payload ,BytesWritable pKey) throws IOException {
  if (hasNext()) {

    MessageAndOffset msgAndOffset = messageIter.next();
    Message message = msgAndOffset.message();

    ByteBuffer buf = message.payload();
    int origSize = buf.remaining();
    byte[] bytes = new byte[origSize];
    buf.get(bytes, buf.position(), origSize);
    payload.set(bytes, 0, origSize);

    buf = message.key();
    if(buf != null){
      origSize = buf.remaining();
      bytes = new byte[origSize];
      buf.get(bytes, buf.position(), origSize);
      pKey.set(bytes, 0, origSize);
    }

    key.clear();
    key.set(kafkaRequest.getTopic(), kafkaRequest.getLeaderId(),
        kafkaRequest.getPartition(), currentOffset,
        msgAndOffset.offset() + 1, message.checksum());

    key.setMessageSize(msgAndOffset.message().size());

    currentOffset = msgAndOffset.offset() + 1; // increase offset
    currentCount++; // increase count

    return true;
  } else {
    return false;
  }
}
 

@Override
public void run()
{
  Map<String, Integer> topicCountMap = new HashMap<String, Integer>();
  topicCountMap.put(topic, new Integer(1));
  Map<String, List<KafkaStream<byte[], byte[]>>> consumerMap = consumer.createMessageStreams(topicCountMap);
  KafkaStream<byte[], byte[]> stream = consumerMap.get(topic).get(0);
  ConsumerIterator<byte[], byte[]> it = stream.iterator();
  logger.debug("Inside consumer::run receiveCount= {}", receiveCount);
  while (it.hasNext() & isAlive) {
    Message msg = new Message(it.next().message());
    if (latch != null) {
      latch.countDown();
    }
    if (getMessage(msg).equals(KafkaOperatorTestBase.END_TUPLE)) {
      break;
    }
    holdingBuffer.add(msg);
    receiveCount++;
    logger.debug("Consuming {}, receiveCount= {}", getMessage(msg), receiveCount);
    try {
      Thread.sleep(50);
    } catch (InterruptedException e) {
      break;
    }
  }
  logger.debug("DONE consuming");
}
 

public String getMessage(Message message)
{
  ByteBuffer buffer = message.payload();
  byte[] bytes = new byte[buffer.remaining()];
  buffer.get(bytes);
  return new String(bytes);
}
 

/**
   * Implement abstract method of AbstractKafkaSinglePortInputOperator
   *
   * @param message
   * @return byte Array
   */
@Override
public byte[] getTuple(Message message)
{
  byte[] bytes = null;
  try {
    ByteBuffer buffer = message.payload();
    bytes = new byte[buffer.remaining()];
    buffer.get(bytes);
  } catch (Exception ex) {
    return bytes;
  }
  return bytes;
}
 

SearchOffset(TopicAndPartition tap, Function<Message, V> factory, V target, long startOffset, long endOffset) {
   this.tap = tap;
   this.factory = factory;
   this.target = target;
   this.startOffset = startOffset;
   this.endOffset = endOffset;
}
 

private void initializeLastProcessingOffset()
{
  // read last received kafka message
  TopicMetadata tm = KafkaMetadataUtil.getTopicMetadata(Sets.newHashSet((String)getConfigProperties().get(KafkaMetadataUtil.PRODUCER_PROP_BROKERLIST)), this.getTopic());

  if (tm == null) {
    throw new RuntimeException("Failed to retrieve topic metadata");
  }

  partitionNum = tm.partitionsMetadata().size();

  lastMsgs = new HashMap<Integer, Pair<byte[],byte[]>>(partitionNum);

  for (PartitionMetadata pm : tm.partitionsMetadata()) {

    String leadBroker = pm.leader().host();
    int port = pm.leader().port();
    String clientName = this.getClass().getName().replace('$', '.') + "_Client_" + tm.topic() + "_" + pm.partitionId();
    SimpleConsumer consumer = new SimpleConsumer(leadBroker, port, 100000, 64 * 1024, clientName);

    long readOffset = KafkaMetadataUtil.getLastOffset(consumer, tm.topic(), pm.partitionId(), kafka.api.OffsetRequest.LatestTime(), clientName);

    FetchRequest req = new FetchRequestBuilder().clientId(clientName).addFetch(tm.topic(), pm.partitionId(), readOffset - 1, 100000).build();

    FetchResponse fetchResponse = consumer.fetch(req);
    for (MessageAndOffset messageAndOffset : fetchResponse.messageSet(tm.topic(), pm.partitionId())) {

      Message m = messageAndOffset.message();

      ByteBuffer payload = m.payload();
      ByteBuffer key = m.key();
      byte[] valueBytes = new byte[payload.limit()];
      byte[] keyBytes = new byte[key.limit()];
      payload.get(valueBytes);
      key.get(keyBytes);
      lastMsgs.put(pm.partitionId(), new Pair<byte[], byte[]>(keyBytes, valueBytes));
    }
  }
}
 

public static Iterable<List<Object>> generateTuples(KafkaConfig kafkaConfig, Message msg) {
    Iterable<List<Object>> tups;
    ByteBuffer payload = msg.payload();
    ByteBuffer key = msg.key();
    if (key != null && kafkaConfig.scheme instanceof KeyValueSchemeAsMultiScheme) {
        tups = ((KeyValueSchemeAsMultiScheme) kafkaConfig.scheme).deserializeKeyAndValue(Utils.toByteArray(key), Utils.toByteArray(payload));
    } else {
        tups = kafkaConfig.scheme.deserialize(Utils.toByteArray(payload));
    }
    return tups;
}
 

public void testIt() throws Exception
{
	Client client = createMock(Client.class);
	IndexRequestBuilder irb = createMock(IndexRequestBuilder.class);
	JsonMessageHandler h = new JsonMessageHandler(client);
	byte[] json = toJson(rec).getBytes();
			
	expect(client.prepareIndex(anyObject(String.class), anyObject(String.class), anyObject(String.class))).andReturn(irb);
	replay(client);
	
	Message message = createMock(Message.class);
	expect(message.payload()).andReturn(ByteBuffer.wrap(json));
	replay(message);
	
	BulkRequestBuilder bulkRequestBuilder = createMock(BulkRequestBuilder.class);
	
	expect(bulkRequestBuilder.add(anyObject(IndexRequestBuilder.class))).andReturn(null);
	replay(bulkRequestBuilder);
	
	try {
		h.handle(bulkRequestBuilder, message);
	} catch (Exception e) {
		fail("This should not fail");
	}
	
	verify(client);
}
 

private <C extends Comparable<? super C>> Function<Message, C> wrapMapGetter(Function<Map.Entry<K, V>, C> delegate) {
   return (message) -> {
      K key = keyDeserializer.apply(message.key());
      V value = valueDeserializer.apply(message.buffer());
      return delegate.apply(new ImmutableMapEntry<>(key, value));
   };
}
 

public PulsarMessageAndMetadata(String topic, int partition, Message rawMessage, long offset, Decoder<K> keyDecoder,
        Decoder<V> valueDecoder, K key, V value) {
    super(topic, partition, rawMessage, offset, keyDecoder, valueDecoder);
    this.topic = topic;
    this.partition = partition;
    this.offset = offset;
    this.keyDecoder = keyDecoder;
    this.valueDecoder = valueDecoder;
    this.key = key;
    this.value = value;
}
 

byte[] getCurrentMessagePayload() {
  while(currentMessageSetIterator.hasNext()) {
    MessageAndOffset messageAndOffset = currentMessageSetIterator.next();
    if (messageAndOffset.offset() < currentOffset) continue; //old offset, ignore
    Message message = messageAndOffset.message();
    ByteBuffer payload = message.payload();
    byte[] bytes = new byte[payload.limit()];
    payload.get(bytes);
    currentOffset = messageAndOffset.nextOffset();
    return bytes;
  }
  return null;
}
 

private void emit(TridentCollector collector, Message msg) {
    Iterable<List<Object>> values = KafkaUtils.generateTuples(_config, msg);
    if (values != null) {
        for (List<Object> value : values) {
            collector.emit(value);
        }
    }
}
 

private static MessageAndMetadata<byte[], byte[]> generateKafkaMessage() {
    byte[] message = "aMessage".getBytes();

    return new MessageAndMetadata<byte[], byte[]>("topic", 0, new Message(message),
            0, new DefaultDecoder(null), new DefaultDecoder(null));
}
 

public void handle(BulkRequestBuilder bulkRequestBuilder, Message message) throws Exception
{
	byte[] data = getMessageData(message);
	bulkRequestBuilder.add(data, 0, data.length, false);
}
 

public MessageAndRealOffset(Message msg, long offset) {
    this.msg = msg;
    this.offset = offset;
}
 

@Override
public void handle(BulkRequestBuilder bulkRequestBuilder, Message message) throws Exception {
	this.readMessage(message);
	bulkRequestBuilder.add( this.createIndexRequestBuilder() );
}
 

protected void readMessage(Message message) throws Exception {
	messageMap = reader.readValue(getMessageData(message));
}
 

public static byte[] getMessageData(Message message) {
	ByteBuffer buf = message.payload();
	byte[] data = new byte[buf.remaining()];
	buf.get(data);
	return data;
}
 

public Message getMsg()
{
  return msg;
}
 

public SearchOffsets(Function<Message, V> factory, V target) {
   this.comparisonBuilder = factory;
   this.target = target;
   this.offsets = new LinkedHashMap<>();
}
 

public ScanSearch(Function<Message, V> getter, V target) {
   this.getter = getter;
   this.target = target;
}