Java Code Examples for org.apache.samza.system.SystemAdmin#createStream()

The following examples show how to use org.apache.samza.system.SystemAdmin#createStream() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: StreamManager.java From samza with Apache License 2.0

6 votes

public void createStreams(List<StreamSpec> streams) {
  Multimap<String, StreamSpec> streamsGroupedBySystem = HashMultimap.create();
  streams.forEach(streamSpec ->
    streamsGroupedBySystem.put(streamSpec.getSystemName(), streamSpec));

  for (Map.Entry<String, Collection<StreamSpec>> entry : streamsGroupedBySystem.asMap().entrySet()) {
    String systemName = entry.getKey();
    SystemAdmin systemAdmin = systemAdmins.getSystemAdmin(systemName);

    for (StreamSpec stream : entry.getValue()) {
      LOGGER.info("Creating stream {} with partitions {} on system {}",
          new Object[]{stream.getPhysicalName(), stream.getPartitionCount(), systemName});
      systemAdmin.createStream(stream);
    }
  }
}

Example 2

Source File: TestKafkaSystemAdminJava.java From samza with Apache License 2.0

5 votes

@Test
public void testCreateCoordinatorStream() {
  SystemAdmin admin = Mockito.spy(systemAdmin());
  StreamSpec spec = StreamSpec.createCoordinatorStreamSpec("testCoordinatorStream", "testSystem");

  admin.createStream(spec);
  admin.validateStream(spec);
  Mockito.verify(admin).createStream(Mockito.any());
}

Example 3

Source File: DiagnosticsUtil.java From samza with Apache License 2.0

5 votes

public static void createDiagnosticsStream(Config config) {
  if (!new JobConfig(config).getDiagnosticsEnabled()) {
    return;
  }
  // if diagnostics is enabled, create diagnostics stream if it doesnt exist

  SystemAdmins systemAdmins = new SystemAdmins(config);
  String diagnosticsSystemStreamName = new MetricsConfig(config)
      .getMetricsSnapshotReporterStream(MetricsConfig.METRICS_SNAPSHOT_REPORTER_NAME_FOR_DIAGNOSTICS)
      .orElseThrow(() -> new ConfigException("Missing required config: " +
          String.format(MetricsConfig.METRICS_SNAPSHOT_REPORTER_STREAM,
              MetricsConfig.METRICS_SNAPSHOT_REPORTER_NAME_FOR_DIAGNOSTICS)));

  SystemStream diagnosticsSystemStream = StreamUtil.getSystemStreamFromNames(diagnosticsSystemStreamName);
  SystemAdmin diagnosticsSysAdmin = systemAdmins.getSystemAdmin(diagnosticsSystemStream.getSystem());
  StreamSpec diagnosticsStreamSpec = new StreamSpec(DIAGNOSTICS_STREAM_ID, diagnosticsSystemStream.getStream(),
      diagnosticsSystemStream.getSystem(), new StreamConfig(config).getStreamProperties(DIAGNOSTICS_STREAM_ID));

  log.info("Creating diagnostics stream {}", diagnosticsSystemStream.getStream());
  diagnosticsSysAdmin.start();

  if (diagnosticsSysAdmin.createStream(diagnosticsStreamSpec)) {
    log.info("Created diagnostics stream {}", diagnosticsSystemStream.getStream());
  } else {
    log.info("Diagnostics stream {} already exists", diagnosticsSystemStream.getStream());
  }

  diagnosticsSysAdmin.stop();
}

Example 4

Source File: StreamAppender.java From samza with Apache License 2.0

4 votes

protected void setupSystem() {
  config = getConfig();
  Log4jSystemConfig log4jSystemConfig = new Log4jSystemConfig(config);

  if (streamName == null) {
    streamName = getStreamName(log4jSystemConfig.getJobName(), log4jSystemConfig.getJobId());
  }

  // TODO we need the ACTUAL metrics registry, or the metrics won't get reported by the metric reporters!
  MetricsRegistry metricsRegistry = new MetricsRegistryMap();
  metrics = new StreamAppenderMetrics("stream-appender", metricsRegistry);

  String systemName = log4jSystemConfig.getSystemName();
  String systemFactoryName = log4jSystemConfig.getSystemFactory(systemName)
      .orElseThrow(() -> new SamzaException(
          "Could not figure out \"" + systemName + "\" system factory for log4j StreamAppender to use"));
  SystemFactory systemFactory = ReflectionUtil.getObj(systemFactoryName, SystemFactory.class);

  setSerde(log4jSystemConfig, systemName, streamName);

  if (config.getBoolean(CREATE_STREAM_ENABLED, false)) {
    // Explicitly create stream appender stream with the partition count the same as the number of containers.
    System.out.println("[StreamAppender] creating stream " + streamName + " with partition count " + getPartitionCount());
    StreamSpec streamSpec =
        StreamSpec.createStreamAppenderStreamSpec(streamName, systemName, getPartitionCount());

    // SystemAdmin only needed for stream creation here.
    SystemAdmin systemAdmin = systemFactory.getAdmin(systemName, config);
    systemAdmin.start();
    systemAdmin.createStream(streamSpec);
    systemAdmin.stop();
  }

  systemProducer = systemFactory.getProducer(systemName, config, metricsRegistry);
  systemStream = new SystemStream(systemName, streamName);
  systemProducer.register(SOURCE);
  systemProducer.start();

  log.info(SOURCE + " has been registered in " + systemName + ". So all the logs will be sent to " + streamName
      + " in " + systemName + ". Logs are partitioned by " + key);

  startTransferThread();
}

Example 5

Source File: StreamAppender.java From samza with Apache License 2.0

4 votes

protected void setupSystem() {
  config = getConfig();
  Log4jSystemConfig log4jSystemConfig = new Log4jSystemConfig(config);

  if (streamName == null) {
    streamName = getStreamName(log4jSystemConfig.getJobName(), log4jSystemConfig.getJobId());
  }

  // TODO we need the ACTUAL metrics registry, or the metrics won't get reported by the metric reporters!
  MetricsRegistry metricsRegistry = new MetricsRegistryMap();
  metrics = new StreamAppenderMetrics("stream-appender", metricsRegistry);

  String systemName = log4jSystemConfig.getSystemName();
  String systemFactoryName = log4jSystemConfig.getSystemFactory(systemName)
      .orElseThrow(() -> new SamzaException(
          "Could not figure out \"" + systemName + "\" system factory for log4j StreamAppender to use"));
  SystemFactory systemFactory = ReflectionUtil.getObj(systemFactoryName, SystemFactory.class);

  setSerde(log4jSystemConfig, systemName, streamName);

  if (config.getBoolean(CREATE_STREAM_ENABLED, false)) {
    // Explicitly create stream appender stream with the partition count the same as the number of containers.
    System.out.println("[StreamAppender] creating stream " + streamName + " with partition count " + getPartitionCount());
    StreamSpec streamSpec = StreamSpec.createStreamAppenderStreamSpec(streamName, systemName, getPartitionCount());

    // SystemAdmin only needed for stream creation here.
    SystemAdmin systemAdmin = systemFactory.getAdmin(systemName, config);
    systemAdmin.start();
    systemAdmin.createStream(streamSpec);
    systemAdmin.stop();
  }

  systemProducer = systemFactory.getProducer(systemName, config, metricsRegistry);
  systemStream = new SystemStream(systemName, streamName);
  systemProducer.register(SOURCE);
  systemProducer.start();

  log.info(SOURCE + " has been registered in " + systemName + ". So all the logs will be sent to " + streamName
      + " in " + systemName + ". Logs are partitioned by " + key);

  startTransferThread();
}