Java Code Examples for org.apache.samza.system.SystemFactory#getConsumer()

The following examples show how to use org.apache.samza.system.SystemFactory#getConsumer() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: CoordinatorStreamSystemConsumer.java    From samza with Apache License 2.0 5 votes vote down vote up 
public CoordinatorStreamSystemConsumer(Config config, MetricsRegistry registry) {
  SystemStream coordinatorSystemStream = CoordinatorStreamUtil.getCoordinatorSystemStream(config);
  SystemFactory systemFactory = CoordinatorStreamUtil.getCoordinatorSystemFactory(config);
  SystemAdmin systemAdmin = systemFactory.getAdmin(coordinatorSystemStream.getSystem(), config);
  SystemConsumer systemConsumer = systemFactory.getConsumer(coordinatorSystemStream.getSystem(), config, registry);

  this.coordinatorSystemStreamPartition = new SystemStreamPartition(coordinatorSystemStream, new Partition(0));
  this.systemConsumer = systemConsumer;
  this.systemAdmin = systemAdmin;
  this.configMap = new HashMap<>();
  this.isBootstrapped = false;
  this.keySerde = new JsonSerde<>();
  this.messageSerde = new JsonSerde<>();
}
Example 2
Source File: CoordinatorStreamStore.java    From samza with Apache License 2.0 5 votes vote down vote up 
public CoordinatorStreamStore(Config config, MetricsRegistry metricsRegistry) {
  this.config = config;
  this.coordinatorSystemStream = CoordinatorStreamUtil.getCoordinatorSystemStream(config);
  this.coordinatorSystemStreamPartition = new SystemStreamPartition(coordinatorSystemStream, new Partition(0));
  SystemFactory systemFactory = CoordinatorStreamUtil.getCoordinatorSystemFactory(config);
  this.systemProducer = systemFactory.getProducer(this.coordinatorSystemStream.getSystem(), config, metricsRegistry);
  this.systemConsumer = systemFactory.getConsumer(this.coordinatorSystemStream.getSystem(), config, metricsRegistry);
  this.systemAdmin = systemFactory.getAdmin(this.coordinatorSystemStream.getSystem(), config);
}
Example 3
Source File: TestRunner.java    From samza with Apache License 2.0 4 votes vote down vote up 
/**
 * Gets the contents of the output stream represented by {@code outputDescriptor} after {@link TestRunner#run(Duration)}
 * has completed
 *
 * @param outputDescriptor describes the stream to be consumed
 * @param timeout timeout for consumption of stream in Ms
 * @param <StreamMessageType> type of message
 *
 * @return a map whose key is {@code partitionId} and value is messages in partition
 * @throws SamzaException Thrown when a poll is incomplete
 */
public static <StreamMessageType> Map<Integer, List<StreamMessageType>> consumeStream(
    InMemoryOutputDescriptor outputDescriptor, Duration timeout) throws SamzaException {
  Preconditions.checkNotNull(outputDescriptor);
  String streamId = outputDescriptor.getStreamId();
  String systemName = outputDescriptor.getSystemName();
  Set<SystemStreamPartition> ssps = new HashSet<>();
  Set<String> streamIds = new HashSet<>();
  streamIds.add(streamId);
  SystemFactory factory = new InMemorySystemFactory();
  Config config = new MapConfig(outputDescriptor.toConfig(), outputDescriptor.getSystemDescriptor().toConfig());
  Map<String, SystemStreamMetadata> metadata = factory.getAdmin(systemName, config).getSystemStreamMetadata(streamIds);
  SystemConsumer consumer = factory.getConsumer(systemName, config, null);
  String name = (String) outputDescriptor.getPhysicalName().orElse(streamId);
  metadata.get(name).getSystemStreamPartitionMetadata().keySet().forEach(partition -> {
    SystemStreamPartition temp = new SystemStreamPartition(systemName, streamId, partition);
    ssps.add(temp);
    consumer.register(temp, "0");
  });

  long t = System.currentTimeMillis();
  Map<SystemStreamPartition, List<IncomingMessageEnvelope>> output = new HashMap<>();
  HashSet<SystemStreamPartition> didNotReachEndOfStream = new HashSet<>(ssps);
  while (System.currentTimeMillis() < t + timeout.toMillis()) {
    Map<SystemStreamPartition, List<IncomingMessageEnvelope>> currentState = null;
    try {
      currentState = consumer.poll(ssps, 10);
    } catch (InterruptedException e) {
      throw new SamzaException("Timed out while consuming stream \n" + e.getMessage());
    }
    for (Map.Entry<SystemStreamPartition, List<IncomingMessageEnvelope>> entry : currentState.entrySet()) {
      SystemStreamPartition ssp = entry.getKey();
      output.computeIfAbsent(ssp, k -> new LinkedList<IncomingMessageEnvelope>());
      List<IncomingMessageEnvelope> currentBuffer = entry.getValue();
      int totalMessagesToFetch = Integer.valueOf(metadata.get(outputDescriptor.getStreamId())
          .getSystemStreamPartitionMetadata()
          .get(ssp.getPartition())
          .getUpcomingOffset());
      if (output.get(ssp).size() + currentBuffer.size() == totalMessagesToFetch) {
        didNotReachEndOfStream.remove(entry.getKey());
        ssps.remove(entry.getKey());
      }
      output.get(ssp).addAll(currentBuffer);
    }
    if (didNotReachEndOfStream.isEmpty()) {
      break;
    }
  }

  if (!didNotReachEndOfStream.isEmpty()) {
    throw new IllegalStateException("Could not poll for all system stream partitions");
  }

  return output.entrySet()
      .stream()
      .collect(Collectors.toMap(entry -> entry.getKey().getPartition().getPartitionId(),
        entry -> entry.getValue().stream().map(e -> (StreamMessageType) e.getMessage()).collect(Collectors.toList())));
}