Java Code Examples for org.apache.samza.system.SystemFactory#getProducer()

/** The dummy stream created will only be used in Beam tests. */
private static InputDescriptor<OpMessage<String>, ?> createDummyStreamDescriptor(String id) {
  final GenericSystemDescriptor dummySystem =
      new GenericSystemDescriptor(id, InMemorySystemFactory.class.getName());
  final GenericInputDescriptor<OpMessage<String>> dummyInput =
      dummySystem.getInputDescriptor(id, new NoOpSerde<>());
  dummyInput.withOffsetDefault(SystemStreamMetadata.OffsetType.OLDEST);
  final Config config = new MapConfig(dummyInput.toConfig(), dummySystem.toConfig());
  final SystemFactory factory = new InMemorySystemFactory();
  final StreamSpec dummyStreamSpec = new StreamSpec(id, id, id, 1);
  factory.getAdmin(id, config).createStream(dummyStreamSpec);

  final SystemProducer producer = factory.getProducer(id, config, null);
  final SystemStream sysStream = new SystemStream(id, id);
  final Consumer<Object> sendFn =
      (msg) -> {
        producer.send(id, new OutgoingMessageEnvelope(sysStream, 0, null, msg));
      };
  final WindowedValue<String> windowedValue =
      WindowedValue.timestampedValueInGlobalWindow("dummy", new Instant());

  sendFn.accept(OpMessage.ofElement(windowedValue));
  sendFn.accept(new WatermarkMessage(BoundedWindow.TIMESTAMP_MAX_VALUE.getMillis()));
  sendFn.accept(new EndOfStreamMessage(null));
  return dummyInput;
}
 

public CoordinatorStreamSystemProducer(Config config, MetricsRegistry registry) {
  SystemStream coordinatorSystemStream = CoordinatorStreamUtil.getCoordinatorSystemStream(config);
  SystemFactory systemFactory = CoordinatorStreamUtil.getCoordinatorSystemFactory(config);
  SystemAdmin systemAdmin = systemFactory.getAdmin(coordinatorSystemStream.getSystem(), config);
  SystemProducer systemProducer = systemFactory.getProducer(coordinatorSystemStream.getSystem(), config, registry);
  this.systemStream = coordinatorSystemStream;
  this.systemProducer = systemProducer;
  this.systemAdmin = systemAdmin;
  this.keySerde = new JsonSerde<>();
  this.messageSerde = new JsonSerde<>();
}
 

public CoordinatorStreamStore(Config config, MetricsRegistry metricsRegistry) {
  this.config = config;
  this.coordinatorSystemStream = CoordinatorStreamUtil.getCoordinatorSystemStream(config);
  this.coordinatorSystemStreamPartition = new SystemStreamPartition(coordinatorSystemStream, new Partition(0));
  SystemFactory systemFactory = CoordinatorStreamUtil.getCoordinatorSystemFactory(config);
  this.systemProducer = systemFactory.getProducer(this.coordinatorSystemStream.getSystem(), config, metricsRegistry);
  this.systemConsumer = systemFactory.getConsumer(this.coordinatorSystemStream.getSystem(), config, metricsRegistry);
  this.systemAdmin = systemFactory.getAdmin(this.coordinatorSystemStream.getSystem(), config);
}
 

/**
 * Creates an in memory stream with {@link InMemorySystemFactory} and feeds its partition with stream of messages
 * @param partitionData key of the map represents partitionId and value represents messages in the partition
 * @param descriptor describes a stream to initialize with the in memory system
 */
private <StreamMessageType> void initializeInMemoryInputStream(InMemoryInputDescriptor<?> descriptor,
    Map<Integer, Iterable<StreamMessageType>> partitionData) {
  String systemName = descriptor.getSystemName();
  String streamName = (String) descriptor.getPhysicalName().orElse(descriptor.getStreamId());
  if (this.app instanceof LegacyTaskApplication) {
    // task.inputs is generated using descriptors for Task/StreamApplication, but needs to be generated here
    // for legacy applications that only specify task.class.
    if (configs.containsKey(TaskConfig.INPUT_STREAMS)) {
      configs.put(TaskConfig.INPUT_STREAMS,
          configs.get(TaskConfig.INPUT_STREAMS).concat("," + systemName + "." + streamName));
    } else {
      configs.put(TaskConfig.INPUT_STREAMS, systemName + "." + streamName);
    }
  }
  InMemorySystemDescriptor imsd = (InMemorySystemDescriptor) descriptor.getSystemDescriptor();
  imsd.withInMemoryScope(this.inMemoryScope);
  addConfig(descriptor.toConfig());
  addConfig(descriptor.getSystemDescriptor().toConfig());
  addSerdeConfigs(descriptor);
  StreamSpec spec = new StreamSpec(descriptor.getStreamId(), streamName, systemName, partitionData.size());
  SystemFactory factory = new InMemorySystemFactory();
  Config config = new MapConfig(descriptor.toConfig(), descriptor.getSystemDescriptor().toConfig());
  factory.getAdmin(systemName, config).createStream(spec);
  InMemorySystemProducer producer = (InMemorySystemProducer) factory.getProducer(systemName, config, null);
  SystemStream sysStream = new SystemStream(systemName, streamName);
  partitionData.forEach((partitionId, partition) -> {
    partition.forEach(e -> {
      Object key = e instanceof KV ? ((KV) e).getKey() : null;
      Object value = e instanceof KV ? ((KV) e).getValue() : e;
      if (value instanceof IncomingMessageEnvelope) {
        producer.send((IncomingMessageEnvelope) value);
      } else {
        producer.send(systemName, new OutgoingMessageEnvelope(sysStream, Integer.valueOf(partitionId), key, value));
      }
    });
    producer.send(systemName, new OutgoingMessageEnvelope(sysStream, Integer.valueOf(partitionId), null,
        new EndOfStreamMessage(null)));
  });
}
 

/**
 * Create a pair of DiagnosticsManager and Reporter for the given jobName, jobId, containerId, and execEnvContainerId,
 * if diagnostics is enabled.
 * execEnvContainerId is the ID assigned to the container by the cluster manager (e.g., YARN).
 */
public static Optional<Pair<DiagnosticsManager, MetricsSnapshotReporter>> buildDiagnosticsManager(String jobName,
    String jobId, JobModel jobModel, String containerId, Optional<String> execEnvContainerId, Config config) {

  JobConfig jobConfig = new JobConfig(config);
  MetricsConfig metricsConfig = new MetricsConfig(config);
  Optional<Pair<DiagnosticsManager, MetricsSnapshotReporter>> diagnosticsManagerReporterPair = Optional.empty();

  if (jobConfig.getDiagnosticsEnabled()) {

    // Diagnostics MetricReporter init
    String diagnosticsReporterName = MetricsConfig.METRICS_SNAPSHOT_REPORTER_NAME_FOR_DIAGNOSTICS;
    String diagnosticsFactoryClassName = metricsConfig.getMetricsFactoryClass(diagnosticsReporterName)
        .orElseThrow(() -> new SamzaException(
            String.format("Diagnostics reporter %s missing .class config", diagnosticsReporterName)));
    MetricsReporterFactory metricsReporterFactory =
        ReflectionUtil.getObj(diagnosticsFactoryClassName, MetricsReporterFactory.class);
    MetricsSnapshotReporter diagnosticsReporter =
        (MetricsSnapshotReporter) metricsReporterFactory.getMetricsReporter(diagnosticsReporterName,
            "samza-container-" + containerId, config);

    // DiagnosticsManager init
    ClusterManagerConfig clusterManagerConfig = new ClusterManagerConfig(config);
    int containerMemoryMb = clusterManagerConfig.getContainerMemoryMb();
    int containerNumCores = clusterManagerConfig.getNumCores();
    long maxHeapSizeBytes = Runtime.getRuntime().maxMemory();
    int containerThreadPoolSize = jobConfig.getThreadPoolSize();
    String taskClassVersion = Util.getTaskClassVersion(config);
    String samzaVersion = Util.getSamzaVersion();
    String hostName = Util.getLocalHost().getHostName();
    Optional<String> diagnosticsReporterStreamName =
        metricsConfig.getMetricsSnapshotReporterStream(diagnosticsReporterName);

    if (!diagnosticsReporterStreamName.isPresent()) {
      throw new ConfigException(
          "Missing required config: " + String.format(MetricsConfig.METRICS_SNAPSHOT_REPORTER_STREAM,
              diagnosticsReporterName));
    }
    SystemStream diagnosticsSystemStream = StreamUtil.getSystemStreamFromNames(diagnosticsReporterStreamName.get());

    // Create a SystemProducer for DiagnosticsManager. This producer is used by the DiagnosticsManager
    // to write to the same stream as the MetricsSnapshotReporter called `diagnosticsreporter`.
    Optional<String> diagnosticsSystemFactoryName =
        new SystemConfig(config).getSystemFactory(diagnosticsSystemStream.getSystem());
    if (!diagnosticsSystemFactoryName.isPresent()) {
      throw new SamzaException("Missing factory in config for system " + diagnosticsSystemStream.getSystem());
    }
    SystemFactory systemFactory = ReflectionUtil.getObj(diagnosticsSystemFactoryName.get(), SystemFactory.class);
    SystemProducer systemProducer =
        systemFactory.getProducer(diagnosticsSystemStream.getSystem(), config, new MetricsRegistryMap());

    DiagnosticsManager diagnosticsManager =
        new DiagnosticsManager(jobName, jobId, jobModel.getContainers(), containerMemoryMb, containerNumCores,
            new StorageConfig(config).getNumPersistentStores(), maxHeapSizeBytes, containerThreadPoolSize,
            containerId, execEnvContainerId.orElse(""), taskClassVersion, samzaVersion, hostName,
            diagnosticsSystemStream, systemProducer,
            Duration.ofMillis(new TaskConfig(config).getShutdownMs()), jobConfig.getAutosizingEnabled());

    diagnosticsManagerReporterPair = Optional.of(new ImmutablePair<>(diagnosticsManager, diagnosticsReporter));
  }

  return diagnosticsManagerReporterPair;
}
 

protected void setupSystem() {
  config = getConfig();
  Log4jSystemConfig log4jSystemConfig = new Log4jSystemConfig(config);

  if (streamName == null) {
    streamName = getStreamName(log4jSystemConfig.getJobName(), log4jSystemConfig.getJobId());
  }

  // TODO we need the ACTUAL metrics registry, or the metrics won't get reported by the metric reporters!
  MetricsRegistry metricsRegistry = new MetricsRegistryMap();
  metrics = new StreamAppenderMetrics("stream-appender", metricsRegistry);

  String systemName = log4jSystemConfig.getSystemName();
  String systemFactoryName = log4jSystemConfig.getSystemFactory(systemName)
      .orElseThrow(() -> new SamzaException(
          "Could not figure out \"" + systemName + "\" system factory for log4j StreamAppender to use"));
  SystemFactory systemFactory = ReflectionUtil.getObj(systemFactoryName, SystemFactory.class);

  setSerde(log4jSystemConfig, systemName, streamName);

  if (config.getBoolean(CREATE_STREAM_ENABLED, false)) {
    // Explicitly create stream appender stream with the partition count the same as the number of containers.
    System.out.println("[StreamAppender] creating stream " + streamName + " with partition count " + getPartitionCount());
    StreamSpec streamSpec =
        StreamSpec.createStreamAppenderStreamSpec(streamName, systemName, getPartitionCount());

    // SystemAdmin only needed for stream creation here.
    SystemAdmin systemAdmin = systemFactory.getAdmin(systemName, config);
    systemAdmin.start();
    systemAdmin.createStream(streamSpec);
    systemAdmin.stop();
  }

  systemProducer = systemFactory.getProducer(systemName, config, metricsRegistry);
  systemStream = new SystemStream(systemName, streamName);
  systemProducer.register(SOURCE);
  systemProducer.start();

  log.info(SOURCE + " has been registered in " + systemName + ". So all the logs will be sent to " + streamName
      + " in " + systemName + ". Logs are partitioned by " + key);

  startTransferThread();
}
 

protected void setupSystem() {
  config = getConfig();
  Log4jSystemConfig log4jSystemConfig = new Log4jSystemConfig(config);

  if (streamName == null) {
    streamName = getStreamName(log4jSystemConfig.getJobName(), log4jSystemConfig.getJobId());
  }

  // TODO we need the ACTUAL metrics registry, or the metrics won't get reported by the metric reporters!
  MetricsRegistry metricsRegistry = new MetricsRegistryMap();
  metrics = new StreamAppenderMetrics("stream-appender", metricsRegistry);

  String systemName = log4jSystemConfig.getSystemName();
  String systemFactoryName = log4jSystemConfig.getSystemFactory(systemName)
      .orElseThrow(() -> new SamzaException(
          "Could not figure out \"" + systemName + "\" system factory for log4j StreamAppender to use"));
  SystemFactory systemFactory = ReflectionUtil.getObj(systemFactoryName, SystemFactory.class);

  setSerde(log4jSystemConfig, systemName, streamName);

  if (config.getBoolean(CREATE_STREAM_ENABLED, false)) {
    // Explicitly create stream appender stream with the partition count the same as the number of containers.
    System.out.println("[StreamAppender] creating stream " + streamName + " with partition count " + getPartitionCount());
    StreamSpec streamSpec = StreamSpec.createStreamAppenderStreamSpec(streamName, systemName, getPartitionCount());

    // SystemAdmin only needed for stream creation here.
    SystemAdmin systemAdmin = systemFactory.getAdmin(systemName, config);
    systemAdmin.start();
    systemAdmin.createStream(streamSpec);
    systemAdmin.stop();
  }

  systemProducer = systemFactory.getProducer(systemName, config, metricsRegistry);
  systemStream = new SystemStream(systemName, streamName);
  systemProducer.register(SOURCE);
  systemProducer.start();

  log.info(SOURCE + " has been registered in " + systemName + ". So all the logs will be sent to " + streamName
      + " in " + systemName + ". Logs are partitioned by " + key);

  startTransferThread();
}