Java Code Examples for kafka.message.Message#payload()

/**
 * Implement abstract method of AbstractActiveMQSinglePortInputOperator
 */
@Override
public String getTuple(Message message)
{
  String data = "";
  try {
    ByteBuffer buffer = message.payload();
    byte[] bytes = new byte[buffer.remaining()];
    buffer.get(bytes);
    data = new String(bytes);
    //logger.debug("Consuming {}", data);
  } catch (Exception ex) {
    return data;
  }
  return data;
}
 

private void initializeLastProcessingOffset()
{
  // read last received kafka message
  TopicMetadata tm = KafkaMetadataUtil.getTopicMetadata(Sets.newHashSet((String)getConfigProperties().get(KafkaMetadataUtil.PRODUCER_PROP_BROKERLIST)), this.getTopic());

  if (tm == null) {
    throw new RuntimeException("Failed to retrieve topic metadata");
  }

  partitionNum = tm.partitionsMetadata().size();

  lastMsgs = new HashMap<Integer, Pair<byte[],byte[]>>(partitionNum);

  for (PartitionMetadata pm : tm.partitionsMetadata()) {

    String leadBroker = pm.leader().host();
    int port = pm.leader().port();
    String clientName = this.getClass().getName().replace('$', '.') + "_Client_" + tm.topic() + "_" + pm.partitionId();
    SimpleConsumer consumer = new SimpleConsumer(leadBroker, port, 100000, 64 * 1024, clientName);

    long readOffset = KafkaMetadataUtil.getLastOffset(consumer, tm.topic(), pm.partitionId(), kafka.api.OffsetRequest.LatestTime(), clientName);

    FetchRequest req = new FetchRequestBuilder().clientId(clientName).addFetch(tm.topic(), pm.partitionId(), readOffset - 1, 100000).build();

    FetchResponse fetchResponse = consumer.fetch(req);
    for (MessageAndOffset messageAndOffset : fetchResponse.messageSet(tm.topic(), pm.partitionId())) {

      Message m = messageAndOffset.message();

      ByteBuffer payload = m.payload();
      ByteBuffer key = m.key();
      byte[] valueBytes = new byte[payload.limit()];
      byte[] keyBytes = new byte[key.limit()];
      payload.get(valueBytes);
      key.get(keyBytes);
      lastMsgs.put(pm.partitionId(), new Pair<byte[], byte[]>(keyBytes, valueBytes));
    }
  }
}
 

/**
   * Implement abstract method of AbstractKafkaSinglePortInputOperator
   *
   * @param message
   * @return byte Array
   */
@Override
public byte[] getTuple(Message message)
{
  byte[] bytes = null;
  try {
    ByteBuffer buffer = message.payload();
    bytes = new byte[buffer.remaining()];
    buffer.get(bytes);
  } catch (Exception ex) {
    return bytes;
  }
  return bytes;
}
 

public String getMessage(Message message)
{
  ByteBuffer buffer = message.payload();
  byte[] bytes = new byte[buffer.remaining()];
  buffer.get(bytes);
  return new String(bytes);
}
 

/**
 * Fetches the next Kafka message and stuffs the results into the key and
 * value
 *
 * @param key
 * @param payload
 * @param pKey
 * @return true if there exists more events
 * @throws IOException
 */
public boolean getNext(KafkaKey key, BytesWritable payload ,BytesWritable pKey) throws IOException {
  if (hasNext()) {

    MessageAndOffset msgAndOffset = messageIter.next();
    Message message = msgAndOffset.message();

    ByteBuffer buf = message.payload();
    int origSize = buf.remaining();
    byte[] bytes = new byte[origSize];
    buf.get(bytes, buf.position(), origSize);
    payload.set(bytes, 0, origSize);

    buf = message.key();
    if(buf != null){
      origSize = buf.remaining();
      bytes = new byte[origSize];
      buf.get(bytes, buf.position(), origSize);
      pKey.set(bytes, 0, origSize);
    }

    key.clear();
    key.set(kafkaRequest.getTopic(), kafkaRequest.getLeaderId(),
        kafkaRequest.getPartition(), currentOffset,
        msgAndOffset.offset() + 1, message.checksum());

    key.setMessageSize(msgAndOffset.message().size());

    currentOffset = msgAndOffset.offset() + 1; // increase offset
    currentCount++; // increase count

    return true;
  } else {
    return false;
  }
}
 

byte[] getCurrentMessagePayload() {
  while(currentMessageSetIterator.hasNext()) {
    MessageAndOffset messageAndOffset = currentMessageSetIterator.next();
    if (messageAndOffset.offset() < currentOffset) continue; //old offset, ignore
    Message message = messageAndOffset.message();
    ByteBuffer payload = message.payload();
    byte[] bytes = new byte[payload.limit()];
    payload.get(bytes);
    currentOffset = messageAndOffset.nextOffset();
    return bytes;
  }
  return null;
}
 

@SuppressWarnings("unchecked")
public Iterable<List<Object>> generateTuples(Message msg) {
    Iterable<List<Object>> tups = null;
    ByteBuffer payload = msg.payload();
    if (payload == null) {
        return null;
    }
    tups = Arrays.asList(Utils.tuple(Utils.toByteArray(payload)));
    return tups;
}
 

public static Iterable<List<Object>> generateTuples(KafkaConfig kafkaConfig, Message msg) {
    Iterable<List<Object>> tups;
    ByteBuffer payload = msg.payload();
    ByteBuffer key = msg.key();
    if (key != null && kafkaConfig.scheme instanceof KeyValueSchemeAsMultiScheme) {
        tups = ((KeyValueSchemeAsMultiScheme) kafkaConfig.scheme).deserializeKeyAndValue(Utils.toByteArray(key), Utils.toByteArray(payload));
    } else {
        tups = kafkaConfig.scheme.deserialize(Utils.toByteArray(payload));
    }
    return tups;
}
 

public byte[] fromMessage(Message msg) {
    ByteBuffer bb = msg.payload();
    byte[] result = new byte[bb.remaining()];
    bb.get(result);
    return fromBytes(result);
}
 

public static byte[] getMessageData(Message message) {
	ByteBuffer buf = message.payload();
	byte[] data = new byte[buf.remaining()];
	buf.get(data);
	return data;
}