com.typesafe.config.ConfigParseOptions Java Examples

@Provides
@Singleton
@SuppressFBWarnings("RV_RETURN_VALUE_IGNORED_NO_SIDE_EFFECT")
Config config() {
    Config cfg = ConfigFactory.load();
    String path = System.getProperty("user.dir") + File.separator + ConfigFile;

    File fp = new File(path);
    if (fp.exists()) {
        ConfigParseOptions options = ConfigParseOptions.defaults();
        options.setSyntax(ConfigSyntax.CONF);

        cfg = ConfigFactory.parseFile(fp, options).withFallback(cfg);
    }

    return cfg;
}
 

@SuppressWarnings("rawtypes")
@Override
public void prepare(Map stormConf, Object registrationArgument, TopologyContext context, IErrorReporter errorReporter) {
    this.config = ConfigFactory.parseString((String) registrationArgument, ConfigParseOptions.defaults());

    if (config.hasPath("appId")) {
        baseTags.put("appId", config.getString("appId"));
    }

    if (config.hasPath("siteId")) {
        baseTags.put("siteId", config.getString("siteId"));
    }

    baseTags.put("appExecId", context.getStormId());

    if (config.hasPath(MetricConfigs.METRIC_PREFIX_CONF)) {
        metricNamePrefix = config.getString(MetricConfigs.METRIC_PREFIX_CONF);
    }
}
 

/**
 * Loads the given configuration file using the mapper, falling back to the given resources.
 *
 * @param type      the type to load
 * @param files     the files to load
 * @param resources the resources to which to fall back
 * @param <T>       the resulting type
 * @return the loaded configuration
 */
public <T> T loadFiles(Class<T> type, List<File> files, List<String> resources) {
  if (files == null) {
    return loadResources(type, resources);
  }

  Config config = ConfigFactory.systemProperties();
  for (File file : files) {
    config = config.withFallback(ConfigFactory.parseFile(file, ConfigParseOptions.defaults().setAllowMissing(false)));
  }

  for (String resource : resources) {
    config = config.withFallback(ConfigFactory.load(classLoader, resource));
  }
  return map(checkNotNull(config, "config cannot be null").resolve(), type);
}
 

private static Config loadFromSearchPath(List<String> configSearchPath, String cmdLineConfig,
        ConfigParseOptions opt, List<String> loadedFiles, List<String> loadedFileIndices) {
    Config conf = null;
    
    int index = 1;
    for (String c : configSearchPath) {
        conf = loadOne(c, conf, opt);
        loadedFiles.add(c);
        loadedFileIndices.add(String.valueOf(index++));
    }
    
    if (cmdLineConfig != null) {
        conf = loadOne(cmdLineConfig, conf, opt);
        loadedFiles.add(cmdLineConfig);
        loadedFileIndices.add("R");
    }
    
    return conf;
}
 

/**
 * To use from command line:
 * <p></p>
 * Supply configuration similar to src/test/resources/rss.conf
 * <p></p>
 * Launch using:
 * <p></p>
 * mvn exec:java -Dexec.mainClass=org.apache.streams.rss.provider.RssStreamProvider -Dexec.args="rss.conf articles.json"
 * @param args args
 * @throws Exception Exception
 */
public static void main(String[] args) throws Exception {

  Preconditions.checkArgument(args.length >= 2);

  String configfile = args[0];
  String outfile = args[1];

  Config reference = ConfigFactory.load();
  File file = new File(configfile);
  assert (file.exists());
  Config testResourceConfig = ConfigFactory.parseFileAnySyntax(file, ConfigParseOptions.defaults().setAllowMissing(false));

  Config typesafe = testResourceConfig.withFallback(reference).resolve();

  StreamsConfiguration streamsConfiguration = StreamsConfigurator.detectConfiguration(typesafe);
  SysomosConfiguration config = new ComponentConfigurator<>(SysomosConfiguration.class).detectConfiguration(typesafe, "rss");
  SysomosProvider provider = new SysomosProvider(config);

  ObjectMapper mapper = StreamsJacksonMapper.getInstance();

  PrintStream outStream = new PrintStream(new BufferedOutputStream(new FileOutputStream(outfile)));
  provider.prepare(config);
  provider.startStream();
  do {
    Uninterruptibles.sleepUninterruptibly(streamsConfiguration.getBatchFrequencyMs(), TimeUnit.MILLISECONDS);
    for (StreamsDatum datum : provider.readCurrent()) {
      String json;
      try {
        json = mapper.writeValueAsString(datum.getDocument());
        outStream.println(json);
      } catch (JsonProcessingException ex) {
        System.err.println(ex.getMessage());
      }
    }
  }
  while (provider.isRunning());
  provider.cleanUp();
  outStream.flush();
}
 

/**
 * Return the root configuration loaded from the helios configuration files.
 */
static Config loadConfig() {
  final ConfigResolveOptions resolveOptions = ConfigResolveOptions
      .defaults()
      .setAllowUnresolved(true);

  final ConfigParseOptions parseOptions = ConfigParseOptions.defaults();

  final Config baseConfig = ConfigFactory.load(BASE_CONFIG_FILE, parseOptions, resolveOptions);

  final Config appConfig = ConfigFactory.load(APP_CONFIG_FILE, parseOptions, resolveOptions);

  return appConfig.withFallback(baseConfig);
}
 

/**
 * To use from command line:
 *
 * <p></p>
 * Supply (at least) the following required configuration in application.conf:
 *
 * <p></p>
 * twitter.oauth.consumerKey
 * twitter.oauth.consumerSecret
 * twitter.oauth.accessToken
 * twitter.oauth.accessTokenSecret
 * twitter.info
 *
 * <p></p>
 * Launch using:
 *
 * <p></p>
 * mvn exec:java -Dexec.mainClass=org.apache.streams.twitter.provider.TwitterTimelineProvider -Dexec.args="application.conf tweets.json"
 *
 * @param args args
 * @throws Exception Exception
 */
public static void main(String[] args) throws Exception {

  Preconditions.checkArgument(args.length >= 2);

  String configfile = args[0];
  String outfile = args[1];

  File file = new File(configfile);
  assert (file.exists());

  Config testResourceConfig = ConfigFactory.parseFileAnySyntax(file, ConfigParseOptions.defaults().setAllowMissing(false));
  StreamsConfigurator.addConfig(testResourceConfig);

  StreamsConfiguration streamsConfiguration = StreamsConfigurator.detectConfiguration();
  TwitterTimelineProviderConfiguration config = new ComponentConfigurator<>(TwitterTimelineProviderConfiguration.class).detectConfiguration();
  TwitterTimelineProvider provider = new TwitterTimelineProvider(config);

  ObjectMapper mapper = new StreamsJacksonMapper(Stream.of(TwitterDateTimeFormat.TWITTER_FORMAT).collect(Collectors.toList()));

  PrintStream outStream = new PrintStream(new BufferedOutputStream(new FileOutputStream(outfile)));
  provider.prepare(config);
  provider.startStream();
  do {
    Uninterruptibles.sleepUninterruptibly(streamsConfiguration.getBatchFrequencyMs(), TimeUnit.MILLISECONDS);
    for (StreamsDatum datum : provider.readCurrent()) {
      String json;
      try {
        json = mapper.writeValueAsString(datum.getDocument());
        outStream.println(json);
      } catch (JsonProcessingException ex) {
        System.err.println(ex.getMessage());
      }
    }
  }
  while ( provider.isRunning() );
  provider.cleanUp();
  outStream.flush();
}
 

/**
 * To use from command line:
 *
 * <p></p>
 * Supply (at least) the following required configuration in application.conf:
 *
 * <p></p>
 * twitter.oauth.consumerKey
 * twitter.oauth.consumerSecret
 * twitter.oauth.accessToken
 * twitter.oauth.accessTokenSecret
 * twitter.info
 *
 * <p></p>
 * Launch using:
 *
 * <p></p>
 * mvn exec:java -Dexec.mainClass=org.apache.streams.twitter.provider.TwitterEngagersProvider -Dexec.args="application.conf retweeters.json.txt"
 *
 * @param args args
 * @throws Exception Exception
 */
public static void main(String[] args) throws Exception {

  Preconditions.checkArgument(args.length >= 2);

  String configfile = args[0];
  String outfile = args[1];

  File file = new File(configfile);
  assert (file.exists());

  Config testResourceConfig = ConfigFactory.parseFileAnySyntax(file, ConfigParseOptions.defaults().setAllowMissing(false)).withFallback(StreamsConfigurator.getConfig());
  StreamsConfigurator.addConfig(testResourceConfig);

  PrintStream outStream = new PrintStream(new BufferedOutputStream(new FileOutputStream(outfile)));
  ObjectMapper mapper = StreamsJacksonMapper.getInstance(new StreamsJacksonMapperConfiguration().withDateFormats(Stream.of(TwitterDateTimeFormat.TWITTER_FORMAT).collect(Collectors.toList())));

  TwitterEngagersProviderConfiguration config = new ComponentConfigurator<>(TwitterEngagersProviderConfiguration.class).detectConfiguration();
  TwitterEngagersProvider provider = new TwitterEngagersProvider(config);

  Iterator<User> results = provider.call();

  results.forEachRemaining(d -> {
    try {
      outStream.println(mapper.writeValueAsString(d));
    } catch( Exception e ) {
      LOGGER.warn("Exception", e);
    }
  });

  outStream.flush();
}
 

/**
 * To use from command line:
 *
 * <p></p>
 * Supply (at least) the following required configuration in application.conf:
 *
 * <p></p>
 * twitter.oauth.consumerKey
 * twitter.oauth.consumerSecret
 * twitter.oauth.accessToken
 * twitter.oauth.accessTokenSecret
 * twitter.info
 *
 * <p></p>
 * Launch using:
 *
 * <p></p>
 * mvn exec:java -Dexec.mainClass=org.apache.streams.twitter.provider.TwitterTimelineProvider -Dexec.args="application.conf tweets.json"
 *
 * @param args args
 * @throws Exception Exception
 */
public static void main(String[] args) throws Exception {

  Preconditions.checkArgument(args.length >= 2);

  String configfile = args[0];
  String outfile = args[1];

  File file = new File(configfile);
  assert (file.exists());

  Config testResourceConfig = ConfigFactory.parseFileAnySyntax(file, ConfigParseOptions.defaults().setAllowMissing(false));
  StreamsConfigurator.addConfig(testResourceConfig);

  StreamsConfiguration streamsConfiguration = StreamsConfigurator.detectConfiguration();
  SevenDaySearchProviderConfiguration config = new ComponentConfigurator<>(SevenDaySearchProviderConfiguration.class).detectConfiguration();
  SevenDaySearchProvider provider = new SevenDaySearchProvider(config);

  PrintStream outStream = new PrintStream(new BufferedOutputStream(new FileOutputStream(outfile)));
  ObjectMapper mapper = StreamsJacksonMapper.getInstance(new StreamsJacksonMapperConfiguration().withDateFormats(Stream.of(TwitterDateTimeFormat.TWITTER_FORMAT).collect(Collectors.toList())));

  Iterator<Tweet> results = provider.call();

  results.forEachRemaining(d -> {
    try {
      outStream.println(mapper.writeValueAsString(d));
    } catch( Exception e ) {
      LOGGER.warn("Exception", e);
    }
  });

  outStream.flush();

}
 

/**
 * To use from command line:
 *
 * <p></p>
 * Supply (at least) the following required configuration in application.conf:
 *
 * <p></p>
 * twitter.oauth.consumerKey
 * twitter.oauth.consumerSecret
 * twitter.oauth.accessToken
 * twitter.oauth.accessTokenSecret
 * twitter.info
 *
 * <p></p>
 * Launch using:
 *
 * <p></p>
 * mvn exec:java -Dexec.mainClass=org.apache.streams.twitter.provider.TwitterTimelineProvider -Dexec.args="application.conf tweets.json"
 *
 * @param args args
 * @throws Exception Exception
 */
public static void main(String[] args) throws Exception {

  Preconditions.checkArgument(args.length >= 2);

  String configfile = args[0];
  String outfile = args[1];

  Config reference = ConfigFactory.load();
  File file = new File(configfile);
  assert (file.exists());
  Config testResourceConfig = ConfigFactory.parseFileAnySyntax(file, ConfigParseOptions.defaults().setAllowMissing(false));

  Config typesafe  = testResourceConfig.withFallback(reference).resolve();

  StreamsConfiguration streamsConfiguration = StreamsConfigurator.detectConfiguration(typesafe);
  ThirtyDaySearchProviderConfiguration config = new ComponentConfigurator<>(ThirtyDaySearchProviderConfiguration.class).detectConfiguration();
  ThirtyDaySearchProvider provider = new ThirtyDaySearchProvider(config);

  PrintStream outStream = new PrintStream(new BufferedOutputStream(new FileOutputStream(outfile)));
  ObjectMapper mapper = StreamsJacksonMapper.getInstance(new StreamsJacksonMapperConfiguration().withDateFormats(Stream.of(TwitterDateTimeFormat.TWITTER_FORMAT).collect(Collectors.toList())));

  Iterator<Tweet> results = provider.call();

  results.forEachRemaining(d -> {
    try {
      outStream.println(mapper.writeValueAsString(d));
    } catch( Exception e ) {
      LOGGER.warn("Exception", e);
    }
  });

  outStream.flush();

}
 

/**
 * To use from command line:
 *
 * <p></p>
 * Supply (at least) the following required configuration in application.conf:
 *
 * <p></p>
 * twitter.oauth.consumerKey
 * twitter.oauth.consumerSecret
 * twitter.oauth.accessToken
 * twitter.oauth.accessTokenSecret
 * twitter.info
 *
 * <p></p>
 * Launch using:
 *
 * <p></p>
 * mvn exec:java -Dexec.mainClass=org.apache.streams.twitter.provider.TwitterUserInformationProvider -Dexec.args="application.conf tweets.json"
 *
 * @param args args
 * @throws Exception Exception
 */
public static void main(String[] args) throws Exception {

  Preconditions.checkArgument(args.length >= 2);

  String configfile = args[0];
  String outfile = args[1];

  File file = new File(configfile);
  assert (file.exists());

  Config testResourceConfig = ConfigFactory.parseFileAnySyntax(file, ConfigParseOptions.defaults().setAllowMissing(false));
  StreamsConfigurator.addConfig(testResourceConfig);

  StreamsConfiguration streamsConfiguration = StreamsConfigurator.detectConfiguration();
  TwitterUserInformationConfiguration config = new ComponentConfigurator<>(TwitterUserInformationConfiguration.class).detectConfiguration();
  TwitterUserInformationProvider provider = new TwitterUserInformationProvider(config);

  PrintStream outStream = new PrintStream(new BufferedOutputStream(new FileOutputStream(outfile)));
  provider.prepare(config);
  provider.startStream();
  do {
    Uninterruptibles.sleepUninterruptibly(streamsConfiguration.getBatchFrequencyMs(), TimeUnit.MILLISECONDS);
    for (StreamsDatum datum : provider.readCurrent()) {
      String json;
      try {
        json = MAPPER.writeValueAsString(datum.getDocument());
        outStream.println(json);
      } catch (JsonProcessingException ex) {
        System.err.println(ex.getMessage());
      }
    }
  }
  while ( provider.isRunning());
  provider.cleanUp();
  outStream.flush();
}
 

/**
 * To use from command line:
 *
 * <p></p>
 * Supply (at least) the following required configuration in application.conf:
 *
 * <p></p>
 * twitter.oauth.consumerKey
 * twitter.oauth.consumerSecret
 * twitter.oauth.accessToken
 * twitter.oauth.accessTokenSecret
 * twitter.info
 *
 * <p></p>
 * Launch using:
 *
 * <p></p>
 * mvn exec:java -Dexec.mainClass=org.apache.streams.twitter.provider.TwitterFollowingProvider -Dexec.args="application.conf tweets.json"
 *
 * @param args args
 * @throws Exception Exception
 */
public static void main(String[] args) throws Exception {

  Preconditions.checkArgument(args.length >= 2);

  String configfile = args[0];
  String outfile = args[1];

  File file = new File(configfile);
  assert (file.exists());

  Config configFile = ConfigFactory.parseFileAnySyntax(file, ConfigParseOptions.defaults());
  StreamsConfigurator.addConfig(configFile);

  PrintStream outStream = new PrintStream(new BufferedOutputStream(new FileOutputStream(outfile)));
  ObjectMapper mapper = StreamsJacksonMapper.getInstance(new StreamsJacksonMapperConfiguration().withDateFormats(Stream.of(TwitterDateTimeFormat.TWITTER_FORMAT).collect(Collectors.toList())));

  TwitterFollowingConfiguration config = new ComponentConfigurator<>(TwitterFollowingConfiguration.class).detectConfiguration();
  TwitterFollowingProvider provider = new TwitterFollowingProvider(config);

  Iterator<Follow> results = provider.call();

  results.forEachRemaining(d -> {
    try {
      outStream.println(mapper.writeValueAsString(d));
    } catch( Exception e ) {
      LOGGER.warn("Exception", e);
    }
  });

  outStream.flush();
}
 

/**
 * To use from command line:
 *
 * <p></p>
 * Supply configuration similar to src/test/resources/rss.conf
 *
 * <p></p>
 * Launch using:
 *
 * <p></p>
 * mvn exec:java -Dexec.mainClass=org.apache.streams.rss.provider.RssStreamProvider -Dexec.args="rss.conf articles.json"
 * @param args args
 * @throws Exception Exception
 */
public static void main(String[] args) throws Exception {

  Preconditions.checkArgument(args.length >= 2);

  String configfile = args[0];
  String outfile = args[1];

  Config reference = ConfigFactory.load();
  File file = new File(configfile);
  assert (file.exists());
  Config testResourceConfig = ConfigFactory.parseFileAnySyntax(file, ConfigParseOptions.defaults().setAllowMissing(false));

  Config typesafe  = testResourceConfig.withFallback(reference).resolve();

  StreamsConfiguration streamsConfiguration = StreamsConfigurator.detectConfiguration(typesafe);
  RssStreamConfiguration config = new ComponentConfigurator<>(RssStreamConfiguration.class).detectConfiguration(typesafe, "rss");
  RssStreamProvider provider = new RssStreamProvider(config);

  ObjectMapper mapper = StreamsJacksonMapper.getInstance();

  PrintStream outStream = new PrintStream(new BufferedOutputStream(new FileOutputStream(outfile)));
  provider.prepare(config);
  provider.startStream();
  do {
    Uninterruptibles.sleepUninterruptibly(streamsConfiguration.getBatchFrequencyMs(), TimeUnit.MILLISECONDS);
    for (StreamsDatum datum : provider.readCurrent()) {
      String json;
      try {
        json = mapper.writeValueAsString(datum.getDocument());
        outStream.println(json);
      } catch (JsonProcessingException ex) {
        System.err.println(ex.getMessage());
      }
    }
  }
  while ( provider.isRunning());
  provider.cleanUp();
  outStream.flush();
}
 

private void copyJobConfFromResource() throws IOException {
  Map<String, Config> jobConfigs;
  try (InputStream resourceStream = this.jobConfResourceUrl.openStream()) {
    Reader reader = new InputStreamReader(resourceStream);
    Config rawJobConfig = ConfigFactory.parseReader(reader, ConfigParseOptions.defaults().setSyntax(ConfigSyntax.CONF));
    jobConfigs = overrideJobConfigs(rawJobConfig);
    jobConfigs.forEach((jobName, jobConfig)-> {
      try {
        writeJobConf(jobName, jobConfig);
      } catch (IOException e) {
        log.error("Job " + jobName + " config cannot be written.");
      }
    });
  }
}
 

/**
 * To use from command line:
 *
 * <p></p>
 * Supply (at least) the following required configuration in application.conf:
 *
 * <p></p>
 * instagram.clientKey
 * instagram.usersInfo.authorizedTokens
 * instagram.usersInfo.users
 *
 * <p></p>
 * Launch using:
 *
 * <p></p>
 * mvn exec:java \
 * -Dexec.mainClass=org.apache.streams.instagram.provider.recentmedia.InstagramCommentsProvider \
 * -Dexec.args="application.conf comments.json.txt"
 *
 * @param args args
 * @throws Exception Exception
 */
public static void main(String[] args) throws Exception {

  Preconditions.checkArgument(args.length >= 2);

  String configfile = args[0];
  String outfile = args[1];

  Config reference = ConfigFactory.load();
  File file = new File(configfile);
  assert (file.exists());

  Config conf = ConfigFactory.parseFileAnySyntax(file, ConfigParseOptions.defaults().setAllowMissing(false));
  StreamsConfigurator.addConfig(conf);

  StreamsConfiguration streamsConfiguration = StreamsConfigurator.detectConfiguration();
  InstagramCommentsProviderConfiguration config = new ComponentConfigurator<>(InstagramCommentsProviderConfiguration.class).detectConfiguration();
  InstagramCommentsProvider provider = new InstagramCommentsProvider(config);

  PrintStream outStream = new PrintStream(new BufferedOutputStream(new FileOutputStream(outfile)));
  provider.prepare(config);
  provider.startStream();
  do {
    Uninterruptibles.sleepUninterruptibly(streamsConfiguration.getBatchFrequencyMs(), TimeUnit.MILLISECONDS);
    Iterator<StreamsDatum> iterator = provider.readCurrent().iterator();
    while (iterator.hasNext()) {
      StreamsDatum datum = iterator.next();
      String json;
      try {
        json = MAPPER.writeValueAsString(datum.getDocument());
        outStream.println(json);
      } catch (JsonProcessingException ex) {
        System.err.println(ex.getMessage());
      }
    }
  }
  while ( provider.isRunning());
  provider.cleanUp();
  outStream.flush();
}
 

/**
 * To use from command line:
 *
 * <p></p>
 * Supply (at least) the following required configuration in application.conf:
 *
 * <p></p>
 * instagram.clientKey
 * instagram.usersInfo.authorizedTokens
 * instagram.usersInfo.users
 *
 * <p></p>
 * Launch using:
 *
 * <p></p>
 * mvn exec:java \
 * -Dexec.mainClass=org.apache.streams.instagram.provider.userinfo.InstagramUserInfoProvider \
 * -Dexec.args="application.conf userinfo.json"
 *
 * @param args args
 * @throws Exception Exception
 */
public static void main(String[] args) throws Exception {

  Preconditions.checkArgument(args.length >= 2);

  String configfile = args[0];
  String outfile = args[1];

  Config reference = ConfigFactory.load();
  File file = new File(configfile);
  assert (file.exists());

  Config conf = ConfigFactory.parseFileAnySyntax(file, ConfigParseOptions.defaults().setAllowMissing(false));
  StreamsConfigurator.addConfig(conf);

  StreamsConfiguration streamsConfiguration = StreamsConfigurator.detectConfiguration();
  InstagramUserInfoProviderConfiguration config = new ComponentConfigurator<>(InstagramUserInfoProviderConfiguration.class).detectConfiguration();
  InstagramUserInfoProvider provider = new InstagramUserInfoProvider(config);

  PrintStream outStream = new PrintStream(new BufferedOutputStream(new FileOutputStream(outfile)));
  provider.prepare(config);
  provider.startStream();
  do {
    Uninterruptibles.sleepUninterruptibly(streamsConfiguration.getBatchFrequencyMs(), TimeUnit.MILLISECONDS);
    Iterator<StreamsDatum> iterator = provider.readCurrent().iterator();
    while (iterator.hasNext()) {
      StreamsDatum datum = iterator.next();
      String json;
      try {
        json = MAPPER.writeValueAsString(datum.getDocument());
        outStream.println(json);
      } catch (JsonProcessingException ex) {
        System.err.println(ex.getMessage());
      }
    }
  }
  while ( provider.isRunning());
  provider.cleanUp();
  outStream.flush();
}
 

/**
 * To use from command line:
 *
 * <p></p>
 * Supply (at least) the following required configuration in application.conf:
 *
 * <p></p>
 * instagram.clientKey
 * instagram.usersInfo.authorizedTokens
 * instagram.usersInfo.users
 *
 * <p></p>
 * Launch using:
 *
 * <p></p>
 * mvn exec:java \
 * -Dexec.mainClass=org.apache.streams.instagram.provider.recentmedia.InstagramRecentMediaProvider \
 * -Dexec.args="application.conf media.json.txt"
 *
 * @param args args
 * @throws Exception Exception
 */
public static void main(String[] args) throws Exception {

  Preconditions.checkArgument(args.length >= 2);

  String configfile = args[0];
  String outfile = args[1];

  Config reference = ConfigFactory.load();
  File file = new File(configfile);
  assert (file.exists());

  Config conf = ConfigFactory.parseFileAnySyntax(file, ConfigParseOptions.defaults().setAllowMissing(false));
  StreamsConfigurator.addConfig(conf);

  StreamsConfiguration streamsConfiguration = StreamsConfigurator.detectConfiguration();
  InstagramRecentMediaProviderConfiguration config = new ComponentConfigurator<>(InstagramRecentMediaProviderConfiguration.class).detectConfiguration();
  InstagramRecentMediaProvider provider = new InstagramRecentMediaProvider(config);

  PrintStream outStream = new PrintStream(new BufferedOutputStream(new FileOutputStream(outfile)));
  provider.prepare(config);
  provider.startStream();
  do {
    Uninterruptibles.sleepUninterruptibly(streamsConfiguration.getBatchFrequencyMs(), TimeUnit.MILLISECONDS);
    Iterator<StreamsDatum> iterator = provider.readCurrent().iterator();
    while (iterator.hasNext()) {
      StreamsDatum datum = iterator.next();
      String json;
      try {
        json = MAPPER.writeValueAsString(datum.getDocument());
        outStream.println(json);
      } catch (JsonProcessingException ex) {
        System.err.println(ex.getMessage());
      }
    }
  }
  while ( provider.isRunning());
  provider.cleanUp();
  outStream.flush();
}
 

/**
 * Run FacebookPageProvider from command line.
 * @param args configfile outfile
 * @throws Exception Exception
 */
public static void main(String[] args) throws Exception {

  Preconditions.checkArgument(args.length >= 2);

  String configfile = args[0];
  String outfile = args[1];

  File confFile = new File(configfile);
  assert (confFile.exists());

  Config conf = ConfigFactory.parseFileAnySyntax(confFile, ConfigParseOptions.defaults().setAllowMissing(false));
  StreamsConfigurator.addConfig(conf);

  StreamsConfiguration streamsConfiguration = StreamsConfigurator.detectConfiguration();
  FacebookPageProviderConfiguration config = new ComponentConfigurator<>(FacebookPageProviderConfiguration.class).detectConfiguration();
  FacebookPageProvider provider = new FacebookPageProvider(config);

  PrintStream outStream = new PrintStream(new BufferedOutputStream(new FileOutputStream(outfile)));
  provider.prepare(config);
  provider.startStream();
  do {
    Uninterruptibles.sleepUninterruptibly(streamsConfiguration.getBatchFrequencyMs(), TimeUnit.MILLISECONDS);
    for (StreamsDatum datum : provider.readCurrent()) {
      String json;
      try {
        json = MAPPER.writeValueAsString(datum.getDocument());
        outStream.println(json);
      } catch (JsonProcessingException ex) {
        System.err.println(ex.getMessage());
      }
    }
  }
  while ( provider.isRunning());
  provider.cleanUp();
  outStream.flush();
}
 

/**
 * Run FacebookPageFeedProvider from command line.
 * @param args configfile outfile
 * @throws Exception Exception
 */
public static void main(String[] args) throws Exception {

  Preconditions.checkArgument(args.length >= 2);

  String configfile = args[0];
  String outfile = args[1];

  Config reference = ConfigFactory.load();
  File confFile = new File(configfile);
  assert (confFile.exists());

  Config conf = ConfigFactory.parseFileAnySyntax(confFile, ConfigParseOptions.defaults().setAllowMissing(false));
  StreamsConfigurator.addConfig(conf);

  StreamsConfiguration streamsConfiguration = StreamsConfigurator.detectConfiguration();
  FacebookPageFeedProviderConfiguration config = new ComponentConfigurator<>(FacebookPageFeedProviderConfiguration.class).detectConfiguration();
  FacebookPageFeedProvider provider = new FacebookPageFeedProvider(config);

  PrintStream outStream = new PrintStream(new BufferedOutputStream(new FileOutputStream(outfile)));
  provider.prepare(config);
  provider.startStream();
  do {
    Uninterruptibles.sleepUninterruptibly(streamsConfiguration.getBatchFrequencyMs(), TimeUnit.MILLISECONDS);
    for (StreamsDatum datum : provider.readCurrent()) {
      String json;
      try {
        json = MAPPER.writeValueAsString(datum.getDocument());
        outStream.println(json);
      } catch (JsonProcessingException ex) {
        System.err.println(ex.getMessage());
      }
    }
  }
  while ( provider.isRunning());
  provider.cleanUp();
  outStream.flush();
}
 

/**
 * Retrieve current profile status for a list of accounts.
 * @param args args
 * @throws Exception Exception
 */
public static void main(String[] args) throws Exception {

  MatcherAssert.assertThat(args.length, Matchers.greaterThanOrEqualTo(2));

  String configfile = args[0];
  String outfile = args[1];

  File file = new File(configfile);
  assert (file.exists());

  Config conf = ConfigFactory.parseFileAnySyntax(file, ConfigParseOptions.defaults().setAllowMissing(false));
  StreamsConfigurator.addConfig(conf);

  StreamsConfiguration streamsConfiguration = StreamsConfigurator.detectConfiguration();
  GPlusUserDataProviderConfiguration config = new ComponentConfigurator<>(GPlusUserDataProviderConfiguration.class).detectConfiguration();
  GPlusUserDataProvider provider = new GPlusUserDataProvider(config);

  Gson gson = new Gson();

  PrintStream outStream = new PrintStream(new BufferedOutputStream(new FileOutputStream(outfile)));
  provider.prepare(config);
  provider.startStream();
  do {
    Uninterruptibles.sleepUninterruptibly(streamsConfiguration.getBatchFrequencyMs(), TimeUnit.MILLISECONDS);
    for (StreamsDatum datum : provider.readCurrent()) {
      String json;
      if (datum.getDocument() instanceof String) {
        json = (String) datum.getDocument();
      } else {
        json = gson.toJson(datum.getDocument());
      }
      outStream.println(json);
    }
  }
  while ( provider.isRunning());
  provider.cleanUp();
  outStream.flush();
}
 

/**
 * Retrieve recent activity from a list of accounts.
 * @param args args
 * @throws Exception Exception
 */
public static void main(String[] args) throws Exception {

  Preconditions.checkArgument(args.length >= 2);

  String configfile = args[0];
  String outfile = args[1];

  File file = new File(configfile);
  assert (file.exists());

  Config conf = ConfigFactory.parseFileAnySyntax(file, ConfigParseOptions.defaults().setAllowMissing(false));
  StreamsConfigurator.addConfig(conf);

  StreamsConfiguration streamsConfiguration = StreamsConfigurator.detectConfiguration();
  GPlusUserActivityProviderConfiguration config = new ComponentConfigurator<>(GPlusUserActivityProviderConfiguration.class).detectConfiguration();
  GPlusUserActivityProvider provider = new GPlusUserActivityProvider(config);

  Gson gson = new Gson();

  PrintStream outStream = new PrintStream(new BufferedOutputStream(new FileOutputStream(outfile)));
  provider.prepare(config);
  provider.startStream();
  do {
    Uninterruptibles.sleepUninterruptibly(streamsConfiguration.getBatchFrequencyMs(), TimeUnit.MILLISECONDS);
    for (StreamsDatum datum : provider.readCurrent()) {
      String json;
      if (datum.getDocument() instanceof String) {
        json = (String) datum.getDocument();
      } else {
        json = gson.toJson(datum.getDocument());
      }
      outStream.println(json);
    }
  }
  while ( provider.isRunning());
  provider.cleanUp();
  outStream.flush();
}
 

/**
 * To use from command line:
 *
 * <p></p>
 * Supply configuration similar to src/test/resources/rss.conf
 *
 * <p></p>
 * Launch using:
 *
 * <p></p>
 * mvn exec:java -Dexec.mainClass=org.apache.streams.moreover.MoreoverProvider -Dexec.args="rss.conf articles.json"
 *
 * @param args args
 * @throws Exception Exception
 */
public static void main(String[] args) throws Exception {

  Preconditions.checkArgument(args.length >= 2);

  String configfile = args[0];
  String outfile = args[1];

  Config reference = ConfigFactory.load();
  File file = new File(configfile);
  assert (file.exists());
  Config testResourceConfig = ConfigFactory.parseFileAnySyntax(file, ConfigParseOptions.defaults().setAllowMissing(false));

  Config typesafe  = testResourceConfig.withFallback(reference).resolve();

  StreamsConfiguration streamsConfiguration = StreamsConfigurator.detectConfiguration(typesafe);
  MoreoverConfiguration config = new ComponentConfigurator<>(MoreoverConfiguration.class).detectConfiguration(typesafe, "rss");
  MoreoverProvider provider = new MoreoverProvider(config);

  ObjectMapper mapper = StreamsJacksonMapper.getInstance();

  PrintStream outStream = new PrintStream(new BufferedOutputStream(new FileOutputStream(outfile)));
  provider.prepare(config);
  provider.startStream();
  do {
    Uninterruptibles.sleepUninterruptibly(streamsConfiguration.getBatchFrequencyMs(), TimeUnit.MILLISECONDS);
    for (StreamsDatum datum : provider.readCurrent()) {
      String json;
      try {
        json = mapper.writeValueAsString(datum.getDocument());
        outStream.println(json);
      } catch (JsonProcessingException ex) {
        System.err.println(ex.getMessage());
      }
    }
  }
  while ( provider.isRunning() );
  provider.cleanUp();
  outStream.flush();
}
 

private Config getJobConfig() throws IOException {
  try (InputStream resourceStream = Resources.getResource(JOB_CONF_NAME).openStream()) {
    Reader reader = new InputStreamReader(resourceStream);
    Config rawJobConfig =
        ConfigFactory.parseReader(reader, ConfigParseOptions.defaults().setSyntax(ConfigSyntax.CONF));
    rawJobConfig = rawJobConfig.withFallback(getClusterConfig());

    Config newConfig = ClusterIntegrationTestUtils.buildSleepingJob(JOB_ID, TASK_STATE_FILE, 100L);

    newConfig = newConfig.withValue(SleepingTask.TASK_STATE_FILE_KEY, ConfigValueFactory.fromAnyRef(TASK_STATE_FILE));
    newConfig = newConfig.withFallback(rawJobConfig);
    return newConfig;
  }
}
 

@SuppressFBWarnings("RV_RETURN_VALUE_IGNORED_NO_SIDE_EFFECT")
public void parseConfig() {
    String path = System.getProperty("user.dir") + File.separator + ConfigFile;
    File fp = new File(path);
    if (!fp.exists()) {
        logger.warn("No whitelist found. Every command is allowed to execute on this device!");
        return;
    }

    try {
        StringJoiner joiner = new StringJoiner(", ");

        ConfigParseOptions options = ConfigParseOptions.defaults();
        options.setSyntax(ConfigSyntax.CONF);

        Config cfg = ConfigFactory.parseFile(fp, options);
        Set<String> map = cfg.root().keySet();
        for (String key : map) {
            whitelist.put(key, cfg.getString(key));
            joiner.add(key);
        }

        logger.info("Found whitelist of allowed commands to execute, using it...");
        logger.info("Allowed commands: [" + joiner.toString() + "]");

        isEnabled = true;
    } catch (Exception e) {
        logger.error("Unable to process whitelist file", e);
    }
}
 

private Config loadHoconConfigWithFallback(Path path, Config fallback) throws IOException {
  try (InputStream is = fs.open(path);
       Reader reader = new InputStreamReader(is, Charsets.UTF_8)) {
    return ConfigFactory.parseMap(ImmutableMap.of(ConfigurationKeys.JOB_CONFIG_FILE_PATH_KEY,
        PathUtils.getPathWithoutSchemeAndAuthority(path).toString()))
        .withFallback(ConfigFactory.parseReader(reader, ConfigParseOptions.defaults().setSyntax(ConfigSyntax.CONF)))
        .withFallback(fallback);
  }
}
 

private Config loadHoconConfigAtPath(Path path) throws IOException {
  try (InputStream is = fs.open(path);
      Reader reader = new InputStreamReader(is, Charsets.UTF_8)) {
      return ConfigFactory.parseMap(ImmutableMap.of(ConfigurationKeys.JOB_CONFIG_FILE_PATH_KEY,
          PathUtils.getPathWithoutSchemeAndAuthority(path).toString()))
          .withFallback(ConfigFactory.parseReader(reader, ConfigParseOptions.defaults().setSyntax(ConfigSyntax.CONF)));
  }
}
 

private static Config loadOne(String fileName, Config conf, ConfigParseOptions opt) {
    File f = new File(fileName);
    Config nconf = ConfigFactory.parseFile(f, opt);
    if (conf == null) {
        return nconf;
    }
    else {
        return nconf.withFallback(conf);
    }
}
 

private static Config loadNormal(String cmdLineConfig, ConfigParseOptions opt, 
        List<String> loadedFiles, List<String> loadedFileIndices) {
    Config conf;
    
    conf = loadOne(DIST_CONF, null, opt);
    loadedFiles.add(DIST_CONF);
    loadedFileIndices.add("D");
    
    if (SITE_CONF != null) {
        conf = loadOne(SITE_CONF, conf, opt);
        loadedFiles.add(SITE_CONF);
        loadedFileIndices.add("S");
    }
    
    if (USER_CONF != null) {
        conf = loadOne(USER_CONF, conf, opt);
        loadedFiles.add(USER_CONF);
        loadedFileIndices.add("U");
    }
    
    if (cmdLineConfig == null) {
        File runConf = new File("swift.conf");
        if (runConf.exists()) {
            conf = loadOne(runConf.getPath(), conf, opt);
            loadedFiles.add(runConf.getPath());
            loadedFileIndices.add("R");
        }
    }
    
    if (cmdLineConfig != null) {
        conf = loadOne(cmdLineConfig, conf, opt);
        loadedFiles.add(cmdLineConfig);
        loadedFileIndices.add("R");
    }
    
    return conf;
}
 

@Test
public void testGetOozieConfig() throws Exception {
    String oozieConfigStr = "classification.accessType=oozie_api\nclassification.oozieUrl=http://localhost:11000/oozie\nclassification.filter=status=RUNNING\nclassification.authType=SIMPLE";
    ConfigParseOptions options = ConfigParseOptions.defaults()
            .setSyntax(ConfigSyntax.PROPERTIES)
            .setAllowMissing(false);
    Config config = ConfigFactory.parseString(oozieConfigStr, options);
    config = config.getConfig(EagleConfigConstants.CLASSIFICATION_CONFIG);
    OozieMetadataAccessConfig oozieMetadataAccessConfig = OozieMetadataAccessConfig.config2Entity(config);
    System.out.print(oozieMetadataAccessConfig);
    Assert.assertEquals("oozie_api", oozieMetadataAccessConfig.getAccessType());
    Assert.assertEquals("http://localhost:11000/oozie", oozieMetadataAccessConfig.getOozieUrl());
    Assert.assertEquals("status=RUNNING", oozieMetadataAccessConfig.getFilter());
    Assert.assertEquals("SIMPLE", oozieMetadataAccessConfig.getAuthType());
}
 

@SuppressWarnings( {"serial", "rawtypes"})
@Override
public void prepare(Map stormConf, Object registrationArgument, TopologyContext context, IErrorReporter errorReporter) {
    Config config = ConfigFactory.parseString((String) registrationArgument, ConfigParseOptions.defaults());
    topologyName = config.getString("appId");
    topologyId = context.getStormId();
    metricSystem = MetricSystem.load(config);
    metricSystem.tags(new HashMap<String, Object>() {
        {
            put("appId", topologyName);
            put("stormId", topologyId);
        }
    });
    metricSystem.start();
}