Java Code Examples for org.apache.flume.Context#getSubProperties()

@Override
public void configure(Context context) {
    topic = context.getString(CONF_TOPIC);
    if (topic == null) {
        throw new ConfigurationException("Kafka topic must be specified.");
    }

    writeBody = context.getBoolean(CONF_WRITE_BODY, DEFAULT_WRITE_BODY);

    ImmutableMap<String, String> subProperties = context.getSubProperties(CONF_KAFKA);
    Properties properties = new Properties();
    properties.putAll(subProperties);

    producer = new Producer<String, String>(new ProducerConfig(properties));

    mapper = new ObjectMapper();
}
 

@Override
public void configure(Context context) {

       host = context.getString(RedisConstants.CONF_HOST, DEFAULT_HOST);
       port = context.getInteger(RedisConstants.CONF_PORT, DEFAULT_PORT);
       charset = context.getString(RedisConstants.CONF_CHARSET, DEFAULT_CHARSET);
       String rawChannels = context.getString(CONF_CHANNELS);
       String rawPatterns = context.getString(CONF_PCHANNELS);
       if(null != rawChannels){
           channels = rawChannels.trim().split(",");
           pattern = false;
       } else if (null != rawPatterns){
           patterns = rawPatterns.trim().split(",");
           pattern = true;
       } else {
           throw new RuntimeException("You must set " + CONF_CHANNELS  + " or " + CONF_PCHANNELS + " property.");
       }

       poolProps = context.getSubProperties("pool.");

       log.info("Redis Source Configured");
}
 

@Override
public void configure(Context context) throws ConfigurationException {
  super.configure(context);
  sinks = Arrays.asList(context.getString(
      BasicConfigurationConstants.CONFIG_SINKS).split("\\s+"));
  Map<String, String> params = context.getSubProperties(
      BasicConfigurationConstants.CONFIG_SINK_PROCESSOR_PREFIX);
  processorContext = new Context();
  processorContext.putAll(params);
  SinkProcessorType spType = getKnownSinkProcessor(processorContext.getString(
          BasicConfigurationConstants.CONFIG_TYPE));

  if (spType != null) {
    processorConf =
        (SinkProcessorConfiguration) ComponentConfigurationFactory.create(
            this.getComponentName() + "-processor",
            spType.toString(),
            ComponentType.SINK_PROCESSOR);
    if (processorConf != null) {
      processorConf.setSinks(new HashSet<String>(sinks));
      processorConf.configure(processorContext);
    }
  }
  setConfigured();
}
 

@Override
public void configure(Context context) {
  super.configure(context);

  // use binary writable serialize by default
  writeFormat = context.getString("hdfs.writeFormat",
    SequenceFileSerializerType.Writable.name());
  useRawLocalFileSystem = context.getBoolean("hdfs.useRawLocalFileSystem",
      false);
  serializerContext = new Context(
          context.getSubProperties(SequenceFileSerializerFactory.CTX_PREFIX));
  serializer = SequenceFileSerializerFactory
          .getSerializer(writeFormat, serializerContext);
  logger.info("writeFormat = " + writeFormat + ", UseRawLocalFileSystem = "
      + useRawLocalFileSystem);
}
 

@Override
public void configure(Context context) {
  super.configure(context);

  // use binary writable serialize by default
  writeFormat = context.getString("hdfs.writeFormat",
    SequenceFileSerializerType.Writable.name());
  useRawLocalFileSystem = context.getBoolean("hdfs.useRawLocalFileSystem",
      false);
  serializerContext = new Context(
          context.getSubProperties(SequenceFileSerializerFactory.CTX_PREFIX));
  serializer = SequenceFileSerializerFactory
          .getSerializer(writeFormat, serializerContext);
  logger.info("writeFormat = " + writeFormat + ", UseRawLocalFileSystem = "
      + useRawLocalFileSystem);
}
 

@Override
public void configure(Context context) {
  super.configure(context);

  serializerType = context.getString("serializer", "TEXT");
  useRawLocalFileSystem = context.getBoolean("hdfs.useRawLocalFileSystem",
      false);
  serializerContext =
      new Context(context.getSubProperties(EventSerializer.CTX_PREFIX));
  logger.info("Serializer = " + serializerType + ", UseRawLocalFileSystem = "
      + useRawLocalFileSystem);
}
 

private void initializeSystemProperties(Context context) {
  Map<String, String> sysProps = new HashMap<String, String>();

  Map<String, String> sysPropsOld = context.getSubProperties(
      ConfigurationConstants.OLD_CONFIG_JDBC_SYSPROP_PREFIX);

  if (sysPropsOld.size() > 0) {
    LOGGER.warn("Long form configuration prefix \""
        + ConfigurationConstants.OLD_CONFIG_JDBC_SYSPROP_PREFIX
        + "\" is deprecated. Please use the short form prefix \""
        + ConfigurationConstants.CONFIG_JDBC_SYSPROP_PREFIX
        + "\" instead.");

    sysProps.putAll(sysPropsOld);
  }

  Map<String, String> sysPropsNew = context.getSubProperties(
      ConfigurationConstants.CONFIG_JDBC_SYSPROP_PREFIX);

  // Override the deprecated values with the non-deprecated
  if (sysPropsNew.size() > 0) {
    sysProps.putAll(sysPropsNew);
  }

  for (String key: sysProps.keySet()) {
    String value = sysProps.get(key);
    if(key != null && value != null) {
      System.setProperty(key, value);
    }
  }
}
 

@Test
public void testWithExistingKeyStore() throws Exception {
  keyAliasPassword.putAll(EncryptionTestUtils.
      configureTestKeyStore(baseDir, keyStoreFile));
  Context context = new Context(EncryptionTestUtils.
      configureForKeyStore(keyStoreFile,
          keyStorePasswordFile, keyAliasPassword));
  Context keyProviderContext = new Context(
      context.getSubProperties(EncryptionConfiguration.KEY_PROVIDER + "."));
  KeyProvider keyProvider = KeyProviderFactory.
      getInstance(KeyProviderType.JCEKSFILE.name(), keyProviderContext);
  testKeyProvider(keyProvider);
}
 

@Test
public void testWithNewKeyStore() throws Exception {
  createNewKeyStore();
  EncryptionTestUtils.createKeyStore(keyStoreFile, keyStorePasswordFile,
      keyAliasPassword);
  Context context = new Context(EncryptionTestUtils.
      configureForKeyStore(keyStoreFile,
          keyStorePasswordFile, keyAliasPassword));
  Context keyProviderContext = new Context(
      context.getSubProperties(EncryptionConfiguration.KEY_PROVIDER + "."));
  KeyProvider keyProvider = KeyProviderFactory.
      getInstance(KeyProviderType.JCEKSFILE.name(), keyProviderContext);
  testKeyProvider(keyProvider);
}
 

@Override
public void configure(Context context) {
  super.configure(context);

  serializerType = context.getString("serializer", "TEXT");
  useRawLocalFileSystem = context.getBoolean("hdfs.useRawLocalFileSystem",
      false);
  serializerContext = new Context(
      context.getSubProperties(EventSerializer.CTX_PREFIX));
  logger.info("Serializer = " + serializerType + ", UseRawLocalFileSystem = "
      + useRawLocalFileSystem);
}
 

@Override
public void configure(Context context) {
  Configurables.ensureRequiredNonNull(context,
      SyslogSourceConfigurationConstants.CONFIG_PORT);
  port = context.getInteger(SyslogSourceConfigurationConstants.CONFIG_PORT);
  host = context.getString(SyslogSourceConfigurationConstants.CONFIG_HOST);
  eventSize = context.getInteger("eventSize", SyslogUtils.DEFAULT_SIZE);
  formaterProp = context.getSubProperties(
      SyslogSourceConfigurationConstants.CONFIG_FORMAT_PREFIX);
}
 

@Override
public void configure(Context context) {
  Configurables.ensureRequiredNonNull(
      context, SyslogSourceConfigurationConstants.CONFIG_PORT);
  port = context.getInteger(SyslogSourceConfigurationConstants.CONFIG_PORT);
  host = context.getString(SyslogSourceConfigurationConstants.CONFIG_HOST);
  formaterProp = context.getSubProperties(
      SyslogSourceConfigurationConstants.CONFIG_FORMAT_PREFIX);
}
 

private void configureSerializers(Context context) {
  String serializerListStr = context.getString(SERIALIZERS);
  Preconditions.checkArgument(!StringUtils.isEmpty(serializerListStr),
      "Must supply at least one name and serializer");

  String[] serializerNames = serializerListStr.split("\\s+");

  Context serializerContexts =
      new Context(context.getSubProperties(SERIALIZERS + "."));

  serializerList = Lists.newArrayListWithCapacity(serializerNames.length);
  for(String serializerName : serializerNames) {
    Context serializerContext = new Context(
        serializerContexts.getSubProperties(serializerName + "."));
    String type = serializerContext.getString("type", "DEFAULT");
    String name = serializerContext.getString("name");
    Preconditions.checkArgument(!StringUtils.isEmpty(name),
        "Supplied name cannot be empty.");

    if("DEFAULT".equals(type)) {
      serializerList.add(new NameAndSerializer(name, defaultSerializer));
    } else {
      serializerList.add(new NameAndSerializer(name, getCustomSerializer(
          type, serializerContext)));
    }
  }
}
 

/**
 * Add all configuration parameters starting with "kafka"
 * to consumer properties
 */
private static void setKafkaProps(Context context,Properties kafkaProps) {

  Map<String,String> kafkaProperties =
          context.getSubProperties(KafkaSourceConstants.PROPERTY_PREFIX);

  for (Map.Entry<String,String> prop : kafkaProperties.entrySet()) {

    kafkaProps.put(prop.getKey(), prop.getValue());
    if (log.isDebugEnabled()) {
      log.debug("Reading a Kafka Producer Property: key: "
              + prop.getKey() + ", value: " + prop.getValue());
    }
  }
}
 

@Override
public void configure(Context context) {
  super.configure(context);

  serializerType = context.getString("serializer", "TEXT");
  useRawLocalFileSystem = context.getBoolean("hdfs.useRawLocalFileSystem",
      false);
  serializerContext = new Context(
      context.getSubProperties(EventSerializer.CTX_PREFIX));
  logger.info("Serializer = " + serializerType + ", UseRawLocalFileSystem = "
      + useRawLocalFileSystem);
}
 

@Override
public void configure(Context context) {
  super.configure(context);

  serializerType = context.getString("serializer", "TEXT");
  useRawLocalFileSystem = context.getBoolean("hdfs.useRawLocalFileSystem",
      false);
  serializerContext =
      new Context(context.getSubProperties(EventSerializer.CTX_PREFIX));
  logger.info("Serializer = " + serializerType + ", UseRawLocalFileSystem = "
      + useRawLocalFileSystem);
}
 

@Override
public void configure(Context context) {
  this.headerName = context.getString(CONFIG_MULTIPLEX_HEADER_NAME,
      DEFAULT_MULTIPLEX_HEADER);

  Map<String, Channel> channelNameMap = getChannelNameMap();

  defaultChannels = getChannelListFromNames(
      context.getString(CONFIG_DEFAULT_CHANNEL), channelNameMap);

  Map<String, String> mapConfig =
      context.getSubProperties(CONFIG_PREFIX_MAPPING);

  channelMapping = new HashMap<String, List<Channel>>();

  for (String headerValue : mapConfig.keySet()) {
    List<Channel> configuredChannels = getChannelListFromNames(
        mapConfig.get(headerValue),
        channelNameMap);

    //This should not go to default channel(s)
    //because this seems to be a bad way to configure.
    if (configuredChannels.size() == 0) {
      throw new FlumeException("No channel configured for when "
          + "header value is: " + headerValue);
    }

    if (channelMapping.put(headerValue, configuredChannels) != null) {
      throw new FlumeException("Selector channel configured twice");
    }
  }
  //If no mapping is configured, it is ok.
  //All events will go to the default channel(s).
  Map<String, String> optionalChannelsMapping =
      context.getSubProperties(CONFIG_PREFIX_OPTIONAL + ".");

  optionalChannels = new HashMap<String, List<Channel>>();
  for (String hdr : optionalChannelsMapping.keySet()) {
    List<Channel> confChannels = getChannelListFromNames(
            optionalChannelsMapping.get(hdr), channelNameMap);
    if (confChannels.isEmpty()) {
      confChannels = EMPTY_LIST;
    }
    //Remove channels from optional channels, which are already
    //configured to be required channels.

    List<Channel> reqdChannels = channelMapping.get(hdr);
    //Check if there are required channels, else defaults to default channels
    if(reqdChannels == null || reqdChannels.isEmpty()) {
      reqdChannels = defaultChannels;
    }
    for (Channel c : reqdChannels) {
      if (confChannels.contains(c)) {
        confChannels.remove(c);
      }
    }

    if (optionalChannels.put(hdr, confChannels) != null) {
      throw new FlumeException("Selector channel configured twice");
    }
  }

}
 

public void configure(Context context) throws ConfigurationException {
  super.configure(context);
  try {
    String channelList = context.getString(
        BasicConfigurationConstants.CONFIG_CHANNELS);
    if (channelList != null) {
      this.channels =
          new HashSet<String>(Arrays.asList(channelList.split("\\s+")));
    }
    if (channels.isEmpty()) {
      errors.add(new FlumeConfigurationError(componentName,
          ComponentType.CHANNEL.getComponentType(),
          FlumeConfigurationErrorType.PROPERTY_VALUE_NULL,
          ErrorOrWarning.ERROR));
      throw new ConfigurationException("No channels set for "
          + this.getComponentName());
    }
    Map<String, String> selectorParams = context.getSubProperties(
            BasicConfigurationConstants.CONFIG_SOURCE_CHANNELSELECTOR_PREFIX);
    String selType;
    if (selectorParams != null && !selectorParams.isEmpty()) {
      selType = selectorParams.get(BasicConfigurationConstants.CONFIG_TYPE);
    } else {
      selType = ChannelSelectorConfigurationType.REPLICATING.toString();
    }

    if (selType == null || selType.isEmpty()) {
      selType = ChannelSelectorConfigurationType.REPLICATING.toString();

    }
    ChannelSelectorType selectorType =
        this.getKnownChannelSelector(selType);
    Context selectorContext = new Context();
    selectorContext.putAll(selectorParams);
    String config = null;
    if (selectorType == null) {
      config = selectorContext.getString(
          BasicConfigurationConstants.CONFIG_CONFIG);
      if (config == null || config.isEmpty()) {
        config = "OTHER";
      }
    } else {
      config = selectorType.toString().toUpperCase();
    }

    this.selectorConf =
        (ChannelSelectorConfiguration) ComponentConfigurationFactory
            .create(ComponentType.CHANNELSELECTOR.getComponentType(), config,
                ComponentType.CHANNELSELECTOR);
    selectorConf.setChannels(channels);
    selectorConf.configure(selectorContext);
  } catch (Exception e) {
    errors.add(new FlumeConfigurationError(componentName,
        ComponentType.CHANNELSELECTOR.getComponentType(),
        FlumeConfigurationErrorType.CONFIG_ERROR,
        ErrorOrWarning.ERROR));
    throw new ConfigurationException("Failed to configure component!", e);
  }
}