Java Code Examples for org.apache.samza.config.Config#getInt()

@Override
public DiskQuotaPolicy create(Config config) {
  final int entryCount = config.getInt(POLICY_COUNT_KEY, 0);
  if (entryCount == 0) {
    log.info("Using a no throttling disk quota policy because policy entry count was missing or set to zero ({})",
        POLICY_COUNT_KEY);
    return new NoThrottlingDiskQuotaPolicy();
  }

  final List<WatermarkDiskQuotaPolicy.Entry> entries = new ArrayList<WatermarkDiskQuotaPolicy.Entry>();
  for (int i = 0; i < entryCount; ++i) {
    final double lowWaterMark = config.getDouble(String.format("container.disk.quota.policy.%d.lowWaterMark", i));
    final double highWaterMark = config.getDouble(String.format("container.disk.quota.policy.%d.highWaterMark", i));
    final double workFactor = config.getDouble(String.format("container.disk.quota.policy.%d.workFactor", i));
    entries.add(new WatermarkDiskQuotaPolicy.Entry(lowWaterMark, highWaterMark, workFactor));
  }

  return new WatermarkDiskQuotaPolicy(entries);
}
 

@Override
public void init(Context context) throws Exception {
  Config config = context.getJobContext().getConfig();
  this.expectedMessageCount = config.getInt("app.messageCount");
  this.outputTopic = config.get("app.outputTopic", "output");
  this.outputSystem = config.get("app.outputSystem", "test-system");
}
 

public TestAvroSystemConsumer(String systemName, Config config) {
  numMessages = config.getInt(String.format("systems.%s.%s", systemName, CFG_NUM_MESSAGES), DEFAULT_NUM_EVENTS);
  includeNullForeignKeys = config.getBoolean(String.format("systems.%s.%s", systemName,
      CFG_INCLUDE_NULL_FOREIGN_KEYS), false);
  includeNullSimpleRecords = config.getBoolean(String.format("systems.%s.%s", systemName,
      CFG_INCLUDE_NULL_SIMPLE_RECORDS), false);
  sleepBetweenPollsMs = config.getLong(String.format("systems.%s.%s", systemName, CFG_SLEEP_BETWEEN_POLLS_MS), 0);
}
 

@Override
public void init(Context context) throws Exception {
  Config config = context.getJobContext().getConfig();
  this.expectedMessageCount = config.getInt("app.messageCount");
  this.outputTopic = config.get("app.outputTopic", "output");
  this.outputSystem = config.get("app.outputSystem", "test-system");
}
 

@Override
public SystemConsumer getConsumer(String systemName, Config config, MetricsRegistry registry) {
  String host = config.get("systems." + systemName + ".host");
  int port = config.getInt("systems." + systemName + ".port");
  WikipediaFeed feed = new WikipediaFeed(host, port);

  return new WikipediaConsumer(systemName, feed, registry);
}
 

/**
 * Constructs a key-value StorageEngine and returns it to the caller
 *
 * @param storeName The name of the storage engine.
 * @param storeDir The directory of the storage engine.
 * @param keySerde The serializer to use for serializing keys when reading or writing to the store.
 * @param msgSerde The serializer to use for serializing messages when reading or writing to the store.
 * @param changelogCollector MessageCollector the storage engine uses to persist changes.
 * @param registry MetricsRegistry to which to publish storage-engine specific metrics.
 * @param changelogSSP Samza system stream partition from which to receive the changelog.
 * @param containerContext Information about the container in which the task is executing.
 **/
public StorageEngine getStorageEngine(String storeName,
    File storeDir,
    Serde<K> keySerde,
    Serde<V> msgSerde,
    MessageCollector changelogCollector,
    MetricsRegistry registry,
    SystemStreamPartition changelogSSP,
    JobContext jobContext,
    ContainerContext containerContext,
    StoreMode storeMode) {
  Config storageConfigSubset = jobContext.getConfig().subset("stores." + storeName + ".", true);
  StorageConfig storageConfig = new StorageConfig(jobContext.getConfig());
  Optional<String> storeFactory = storageConfig.getStorageFactoryClassName(storeName);
  StoreProperties.StorePropertiesBuilder storePropertiesBuilder = new StoreProperties.StorePropertiesBuilder();
  if (!storeFactory.isPresent() || StringUtils.isBlank(storeFactory.get())) {
    throw new SamzaException(
        String.format("Store factory not defined for store %s. Cannot proceed with KV store creation!", storeName));
  }
  if (!storeFactory.get().equals(INMEMORY_KV_STORAGE_ENGINE_FACTORY)) {
    storePropertiesBuilder.setPersistedToDisk(true);
  }
  int batchSize = storageConfigSubset.getInt(WRITE_BATCH_SIZE, DEFAULT_WRITE_BATCH_SIZE);
  int cacheSize = storageConfigSubset.getInt(OBJECT_CACHE_SIZE, Math.max(batchSize, DEFAULT_OBJECT_CACHE_SIZE));
  if (cacheSize > 0 && cacheSize < batchSize) {
    throw new SamzaException(
        String.format("cache.size for store %s cannot be less than batch.size as batched values reside in cache.",
            storeName));
  }
  if (keySerde == null) {
    throw new SamzaException(
        String.format("Must define a key serde when using key value storage for store %s.", storeName));
  }
  if (msgSerde == null) {
    throw new SamzaException(
        String.format("Must define a message serde when using key value storage for store %s.", storeName));
  }

  KeyValueStore<byte[], byte[]> rawStore =
      getKVStore(storeName, storeDir, registry, changelogSSP, jobContext, containerContext, storeMode);
  KeyValueStore<byte[], byte[]> maybeLoggedStore = buildMaybeLoggedStore(changelogSSP,
      storeName, registry, storePropertiesBuilder, rawStore, changelogCollector);
  // this also applies serialization and caching layers
  KeyValueStore<K, V> toBeAccessLoggedStore = buildStoreWithLargeMessageHandling(storeName, registry,
      maybeLoggedStore, storageConfig, cacheSize, batchSize, keySerde, msgSerde);
  KeyValueStore<K, V> maybeAccessLoggedStore =
      buildMaybeAccessLoggedStore(storeName, toBeAccessLoggedStore, changelogCollector, changelogSSP, storageConfig,
          keySerde);
  KeyValueStore<K, V> nullSafeStore = new NullSafeKeyValueStore<>(maybeAccessLoggedStore);

  KeyValueStorageEngineMetrics keyValueStorageEngineMetrics = new KeyValueStorageEngineMetrics(storeName, registry);
  HighResolutionClock clock = buildClock(jobContext.getConfig());
  return new KeyValueStorageEngine<>(storeName, storeDir, storePropertiesBuilder.build(), nullSafeStore, rawStore,
      changelogSSP, changelogCollector, keyValueStorageEngineMetrics, batchSize,
      ScalaJavaUtil.toScalaFunction(clock::nanoTime));
}