Java Code Examples for org.apache.samza.task.MessageCollector#send()

private void processWatermark(long timeStamp, MessageCollector collector) {
    if (timeStamp > lastWatermark + watermarkEvictionPeriod) {
        for (SlicingWindowOperator<Value> slicingWindowOperator : this.slicingWindowOperatorMap.values()) {
            List<AggregateWindow> aggregates = slicingWindowOperator.processWatermark(timeStamp);
            for (AggregateWindow<Value> aggregateWindow : aggregates) {
                if (aggregateWindow.hasValue()) {
                    System.out.println(aggregateWindow);
                    for (Value aggValue : aggregateWindow.getAggValues()) {
                        collector.send(new OutgoingMessageEnvelope(outputStream, aggValue));
                    }

                }
            }
        }
        lastWatermark = timeStamp;
    }
}
 

private void fanOut(String sender, Map<String, Object> message, MessageCollector collector) {
  // Colon is used as separator, and semicolon is lexicographically after colon
  KeyValueIterator<String, String> followers = socialGraph.range(sender + ":", sender + ";");

  try {
    while (followers.hasNext()) {
      String[] follow = followers.next().getKey().split(":");
      if (!follow[0].equals(sender)) {
        throw new IllegalStateException("Social graph db prefix doesn't match: " + sender + " != " + follow[0]);
      }
      message.put("recipient", follow[1]);
      collector.send(new OutgoingMessageEnvelope(NewsfeedConfig.DELIVERIES_STREAM, follow[1], null, message));
    }
  } finally {
    followers.close();
  }
}
 

@Override
public void window(MessageCollector collector,
  TaskCoordinator coordinator) {

  KeyValueIterator<String, String> entries = store.all();
  while (entries.hasNext()) {                                        // c
    Entry<String, String> entry = entries.next();
    String key = entry.getKey();
    String value = entry.getValue();
    if (isTimestampKey(key) && Cart.isAbandoned(value)) {            // d
      String shopper = extractShopper(key);
      String cart = store.get(asCartKey(shopper));
      
      AbandonedCartEvent event =
        new AbandonedCartEvent(shopper, cart);
      collector.send(new OutgoingMessageEnvelope(
        new SystemStream("kafka", "derived-events-ch04"), event));    // e
      
      resetShopper(shopper);
    }
  }
}
 

@Override
public void window(MessageCollector collector, TaskCoordinator coordinator) {
  counts.put("edits", edits);
  counts.put("bytes-added", byteDiff);
  counts.put("unique-titles", titles.size());
  counts.put("edits-all-time", store.get("count-edits-all-time"));

  collector.send(new OutgoingMessageEnvelope(OUTPUT_STREAM, counts));

  // Reset counts after windowing.
  edits = 0;
  byteDiff = 0;
  titles = new HashSet<String>();
  counts = new HashMap<String, Integer>();
}
 

@Override
protected CompletionStage<Collection<M>> handleMessageAsync(M message, MessageCollector collector,
    TaskCoordinator coordinator) {
  Object key, value;
  if (outputStream.isKeyed()) {
    key = ((KV) message).getKey();
    value = ((KV) message).getValue();
  } else {
    key = null;
    value = message;
  }

  collector.send(new OutgoingMessageEnvelope(systemStream, null, key, value));
  return CompletableFuture.completedFuture(Collections.singleton(message));
}
 

void send(ControlMessage message, SystemStream systemStream, MessageCollector collector) {
  int partitionCount = getPartitionCount(systemStream);
  // We pick a partition based on topic hashcode to aggregate the control messages from upstream tasks
  // After aggregation the task will broadcast the results to other partitions
  int aggregatePartition = systemStream.getStream().hashCode() % partitionCount;

  LOG.debug(String.format("Send %s message from task %s to %s partition %s for aggregation",
      MessageType.of(message).name(), message.getTaskName(), systemStream, aggregatePartition));

  OutgoingMessageEnvelope envelopeOut = new OutgoingMessageEnvelope(systemStream, aggregatePartition, null, message);
  collector.send(envelopeOut);
}
 

void broadcastToOtherPartitions(ControlMessage message, SystemStreamPartition ssp, MessageCollector collector) {
  SystemStream systemStream = ssp.getSystemStream();
  int partitionCount = getPartitionCount(systemStream);
  int currentPartition = ssp.getPartition().getPartitionId();
  for (int i = 0; i < partitionCount; i++) {
    if (i != currentPartition) {
      OutgoingMessageEnvelope envelopeOut = new OutgoingMessageEnvelope(systemStream, i, null, message);
      collector.send(envelopeOut);
    }
  }
}
 

@Override
public void window(MessageCollector collector, TaskCoordinator coordinator) {
  for (int i = 0; i < NewsfeedConfig.MESSAGES_PER_WINDOW; i++) {
    String sender = NewsfeedConfig.randomUser();

    HashMap<String, Object> message = new HashMap<String, Object>();
    message.put("event", "postMessage");
    message.put("sender", sender);
    message.put("text", "Hello world");
    message.put("time", NewsfeedConfig.currentDateTime());
    collector.send(new OutgoingMessageEnvelope(NewsfeedConfig.MESSAGES_STREAM, sender, null, message));
  }
}
 

public void window(MessageCollector collector, TaskCoordinator coordinator) {
  Integer epoch = getInt(EPOCH);
  if (epoch == null) {
    resetEpoch();
    return;
  }
  int counter = getInt(COUNT);
  if (counter < max) {
    logger.info("Emitting: " + counter + ", epoch = " + epoch + ", task = " + taskName);
    OutgoingMessageEnvelope envelope = new OutgoingMessageEnvelope(new SystemStream("kafka", "emitted"), Integer.toString(counter), epoch + "-" + taskName.toString());
    collector.send(envelope);
    this.state.put(COUNT, Integer.toString(getInt(COUNT) + 1));
  }
}
 

@Override
public void process(IncomingMessageEnvelope envelope, MessageCollector collector, TaskCoordinator coordinator) {
  String key = (String) envelope.getKey();
  String value = (String) envelope.getMessage();
  String[] pieces = value.split("-");
  int epoch = Integer.parseInt(pieces[0]);

  int partition = Integer.parseInt(pieces[1].split(" ")[1]);
  Partitions partitions = loadPartitions(epoch, key);
  logger.info("Joiner got epoch = " + epoch + ", partition = " + partition + ", parts = " + partitions);
  if (partitions.epoch < epoch) {
    // we are in a new era
    if (partitions.partitions.size() != expected)
      throw new IllegalArgumentException("Should have " + expected + " partitions when new epoch starts.");
    logger.info("Reseting epoch to " + epoch);
    this.store.delete(key);
    partitions.epoch = epoch;
    partitions.partitions.clear();
    partitions.partitions.add(partition);
  } else if (partitions.epoch > epoch) {
    logger.info("Ignoring message for epoch " + epoch);
  } else {
    partitions.partitions.add(partition);
    if (partitions.partitions.size() == expected) {
      logger.info("Completed: " + key + " -> " + Integer.toString(epoch));
      collector.send(new OutgoingMessageEnvelope(new SystemStream("kafka", "completed-keys"), key, Integer.toString(epoch)));
    }
  }
  this.store.put(key, partitions.toString());
  logger.info("Join store in Task " + this.taskName + " " + key + " -> " + partitions.toString());
}
 

@Override
public void process(IncomingMessageEnvelope envelope, MessageCollector collector, TaskCoordinator coordinator) {
  messagesProcessed += 1;
  String input = (String) envelope.getMessage();
  Integer number = Integer.valueOf(input);
  Integer output = number.intValue() * -1;
  collector.send(new OutgoingMessageEnvelope(outputSystemStream, output.toString()));
  if (messagesProcessed >= maxMessages) {
    coordinator.shutdown(RequestScope.ALL_TASKS_IN_CONTAINER);
  }
}
 

@Override
public void process(IncomingMessageEnvelope incomingMessageEnvelope, MessageCollector messageCollector,
    TaskCoordinator taskCoordinator)
    throws Exception {

  Object message = incomingMessageEnvelope.getMessage();

  String key = new String((byte[]) incomingMessageEnvelope.getKey());
  Integer val = Integer.valueOf((String) message);

  LOG.info("Stream processor " + processorId + ";key=" + key + ";offset=" + incomingMessageEnvelope.getOffset()
      + "; totalRcvd=" + processedMessageCount + ";val=" + val + "; ssp=" + incomingMessageEnvelope
      .getSystemStreamPartition());

  // inject a failure
  if (val >= BAD_MESSAGE_KEY && processorId.equals(processorIdToFail)) {
    LOG.info("process method failing for msg=" + message);
    throw new Exception("Processing in the processor " + processorId + " failed ");
  }

  messageCollector.send(new OutgoingMessageEnvelope(new SystemStream(outputSystem, outputTopic), message));
  processedMessageCount++;

  synchronized (endLatch) {
    if (Integer.valueOf(key) < BAD_MESSAGE_KEY) {
      endLatch.countDown();
    }
  }
}
 

@Override
public void process(
    IncomingMessageEnvelope incomingMessageEnvelope,
    MessageCollector messageCollector,
    TaskCoordinator taskCoordinator) throws Exception {
  messageCollector.send(
      new OutgoingMessageEnvelope(
          new SystemStream(outputSystem, outputTopic),
          incomingMessageEnvelope.getMessage()));
  processedMessageCount++;
  if (processedMessageCount == expectedMessageCount) {
    taskCoordinator.shutdown(TaskCoordinator.RequestScope.ALL_TASKS_IN_CONTAINER);
  }
}
 

@SuppressWarnings("unchecked")
@Override
public void process(IncomingMessageEnvelope envelope, MessageCollector collector, TaskCoordinator coordinator) {
  Map<String, Object> jsonObject = (Map<String, Object>) envelope.getMessage();
  WikipediaFeedEvent event = new WikipediaFeedEvent(jsonObject);

  Map<String, Object> parsedJsonObject = WikipediaParser.parseEvent(event);

  if (parsedJsonObject != null) {
    collector.send(new OutgoingMessageEnvelope(OUTPUT_STREAM, parsedJsonObject));
  }
}
 

@Override
public void process(IncomingMessageEnvelope envelope, MessageCollector collector, TaskCoordinator coordinator) {
  Map<String, Object> outgoingMap = WikipediaFeedEvent.toMap((WikipediaFeedEvent) envelope.getMessage());
  collector.send(new OutgoingMessageEnvelope(OUTPUT_STREAM, outgoingMap));
}
 

private synchronized void sendBroadcast(MessageCollector collector, ContentEvent event) {
  for (int i = 0; i < parallelism; i++) {
    collector.send(new OutgoingMessageEnvelope(this.actualSystemStream, i, null, event));
  }
}
 

private void sendGroupByKey(MessageCollector collector, ContentEvent event) {
  collector.send(new OutgoingMessageEnvelope(this.actualSystemStream, event.getKey(), null, event));
}
 

private synchronized void sendShuffle(MessageCollector collector, ContentEvent event) {
  collector.send(new OutgoingMessageEnvelope(this.actualSystemStream, event));
}
 

private void sendControlMessage(ControlMessage message, MessageCollector collector) {
  OutgoingMessageEnvelope envelopeOut = new OutgoingMessageEnvelope(systemStream, 0, null, message);
  collector.send(envelopeOut);
}
 

@Override
protected CompletionStage<Collection<Void>> handleMessageAsync(M message, MessageCollector collector,
    TaskCoordinator coordinator) {
  collector.send(new OutgoingMessageEnvelope(systemStream, 0, null, message));
  return CompletableFuture.completedFuture(Collections.emptyList());
}