Java Code Examples for org.apache.samza.config.Config#subset()

@Override
public Config buildCoordinatorStreamConfig(Config config) {
  JobConfig jobConfig = new JobConfig(config);
  String jobName = jobConfig.getName().orElseThrow(() -> new ConfigException("Missing required config: job.name"));

  String jobId = jobConfig.getJobId();

  // Build a map with just the system config and job.name/job.id. This is what's required to start the JobCoordinator.
  Map<String, String> map = config.subset(String.format(SystemConfig.SYSTEM_ID_PREFIX, jobConfig.getCoordinatorSystemName()), false);
  Map<String, String> addConfig = new HashMap<>();
  addConfig.put(JobConfig.JOB_NAME, jobName);
  addConfig.put(JobConfig.JOB_ID, jobId);
  addConfig.put(JobConfig.JOB_COORDINATOR_SYSTEM, jobConfig.getCoordinatorSystemName());
  addConfig.put(JobConfig.MONITOR_PARTITION_CHANGE_FREQUENCY_MS, String.valueOf(jobConfig.getMonitorPartitionChangeFrequency()));

  addConfig.putAll(map);
  return new MapConfig(addConfig);
}
 

@Override
public SystemConsumer getConsumer(String systemName, Config config, MetricsRegistry registry) {
  final String streamPrefix = "systems." + systemName;
  final Config scopedConfig = config.subset(streamPrefix + ".", true);

  return new Consumer<T>(
      getBoundedSource(scopedConfig),
      getPipelineOptions(config),
      new SamzaMetricsContainer((MetricsRegistryMap) registry),
      scopedConfig.get("stepName"));
}
 

@Override
public SystemConsumer getConsumer(String systemName, Config config, MetricsRegistry registry) {
  final String streamPrefix = "systems." + systemName;
  final Config scopedConfig = config.subset(streamPrefix + ".", true);
  return new Consumer<T, CheckpointMarkT>(
      getUnboundedSource(scopedConfig),
      getPipelineOptions(config),
      new SamzaMetricsContainer((MetricsRegistryMap) registry),
      scopedConfig.get("stepName"));
}
 

private Map<String, Serde> validateAndGetDeserializedSerdes(Config jobConfig, int numSerdes) {
  Config serializers = jobConfig.subset("serializers.registry.", true);
  // make sure that the serializers deserialize correctly
  SerializableSerde<Serde> serializableSerde = new SerializableSerde<>();
  assertEquals(numSerdes, serializers.size());
  return serializers.entrySet().stream().collect(Collectors.toMap(
    e -> e.getKey().replace(SerializerConfig.SERIALIZED_INSTANCE_SUFFIX, ""),
    e -> serializableSerde.fromBytes(Base64.getDecoder().decode(e.getValue().getBytes()))
  ));
}
 

private void validateStreamSerdeConfigure(String streamId, Config config, Map<String, Serde> deserializedSerdes) {
  Config streamConfig = config.subset(String.format("streams.%s.samza.", streamId));
  String keySerdeName = streamConfig.get("key.serde");
  String valueSerdeName = streamConfig.get("msg.serde");
  assertTrue(String.format("Serialized serdes should contain %s key serde", streamId), deserializedSerdes.containsKey(keySerdeName));
  assertTrue(String.format("Serialized %s key serde should be a StringSerde", streamId), keySerdeName.startsWith(StringSerde.class.getSimpleName()));
  assertTrue(String.format("Serialized serdes should contain %s msg serde", streamId), deserializedSerdes.containsKey(valueSerdeName));
  assertTrue(String.format("Serialized %s msg serde should be a JsonSerdeV2", streamId), valueSerdeName.startsWith(JsonSerdeV2.class.getSimpleName()));
}
 

private void validateTableSerdeConfigure(String tableId, Config config, Map<String, Serde> deserializedSerdes) {
  Config streamConfig = config.subset(String.format("stores.%s.", tableId));
  String keySerdeName = streamConfig.get("key.serde");
  String valueSerdeName = streamConfig.get("msg.serde");
  assertTrue(String.format("Serialized serdes should contain %s key serde", tableId), deserializedSerdes.containsKey(keySerdeName));
  assertTrue(String.format("Serialized %s key serde should be a StringSerde", tableId), keySerdeName.startsWith(StringSerde.class.getSimpleName()));
  assertTrue(String.format("Serialized serdes should contain %s value serde", tableId), deserializedSerdes.containsKey(valueSerdeName));
  assertTrue(String.format("Serialized %s msg serde should be a JsonSerdeV2", tableId), valueSerdeName.startsWith(JsonSerdeV2.class.getSimpleName()));
}
 

private void validateIntermediateStreamConfigure(String streamId, String physicalName, Config config) {
  Config intStreamConfig = config.subset(String.format("streams.%s.", streamId),  true);
  assertEquals("intermediate-system", intStreamConfig.get("samza.system"));
  assertEquals(String.valueOf(Integer.MAX_VALUE), intStreamConfig.get("samza.priority"));
  assertEquals("true", intStreamConfig.get("samza.delete.committed.messages"));
  assertEquals(physicalName, intStreamConfig.get("samza.physical.name"));
  assertEquals("true", intStreamConfig.get("samza.intermediate"));
}
 

public static <T> T initializePlugin(String pluginName, String plugin, Config staticConfig,
    String pluginDomainFormat, BiFunction<Object, Config, T> factoryInvoker) {
  String pluginDomain = String.format(pluginDomainFormat, plugin);
  Config pluginConfig = staticConfig.subset(pluginDomain);
  String factoryName = pluginConfig.getOrDefault(CFG_FACTORY, "");
  Validate.notEmpty(factoryName, String.format("Factory is not set for %s", plugin));
  Object factory = ReflectionUtil.getObj(factoryName, Object.class);
  LOG.info("Instantiating {} using factory {} with props {}", pluginName, factoryName, pluginConfig);
  return factoryInvoker.apply(factory, pluginConfig);
}
 

/**
 *
 * Groups configuration defined in the config object for each of the monitors into a MonitorConfig object
 * @param config contains the entire configuration defined for all the monitors.
 * @return a map of monitorName, {@link MonitorConfig}, where each MonitorConfig object
 * contains all the configuration defined for the monitor named monitorName.
 */
public static Map<String, MonitorConfig> getMonitorConfigs(Config config) {
  Map<String, MonitorConfig> monitorConfigMap = new HashMap<>();
  Config monitorConfig = config.subset(MONITOR_PREFIX);
  for (String monitorName : getMonitorNames(monitorConfig)) {
    monitorConfigMap.put(monitorName,
                         new MonitorConfig(monitorConfig.subset(monitorName + MONITOR_CONFIG_KEY_SEPARATOR)));
  }
  return monitorConfigMap;
}
 

@Override
public SystemAdmin getAdmin(String systemName, Config config) {
  final Config scopedConfig = config.subset("systems." + systemName + ".", true);
  return new Admin<T>(getBoundedSource(scopedConfig), getPipelineOptions(config));
}
 

@Override
public SystemAdmin getAdmin(String systemName, Config config) {
  final Config scopedConfig = config.subset("systems." + systemName + ".", true);
  return new Admin<T, CheckpointMarkT>(
      getUnboundedSource(scopedConfig), getPipelineOptions(config));
}