Java Code Examples for com.typesafe.config.ConfigFactory#parseFileAnySyntax()

The following examples show how to use com.typesafe.config.ConfigFactory#parseFileAnySyntax() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: SprinklrProfilesIT.java    From streams with Apache License 2.0 5 votes vote down vote up 
@BeforeClass(alwaysRun = true)
public void setup() throws Exception {
  File conf = new File(configfile);
  Assert.assertTrue(conf.exists());
  Assert.assertTrue(conf.canRead());
  Assert.assertTrue(conf.isFile());
  Config parsedConfig = ConfigFactory.parseFileAnySyntax(conf);
  StreamsConfigurator.addConfig(parsedConfig);
  config = new ComponentConfigurator<>(SprinklrConfiguration.class).detectConfiguration();
  testsconfig = StreamsConfigurator.getConfig().getConfig("org.apache.streams.sprinklr.config.SprinklrConfiguration");
}
Example 2
Source File: MoreoverProvider.java    From streams with Apache License 2.0 5 votes vote down vote up 
/**
 * To use from command line:
 *
 * <p></p>
 * Supply configuration similar to src/test/resources/rss.conf
 *
 * <p></p>
 * Launch using:
 *
 * <p></p>
 * mvn exec:java -Dexec.mainClass=org.apache.streams.moreover.MoreoverProvider -Dexec.args="rss.conf articles.json"
 *
 * @param args args
 * @throws Exception Exception
 */
public static void main(String[] args) throws Exception {

  Preconditions.checkArgument(args.length >= 2);

  String configfile = args[0];
  String outfile = args[1];

  Config reference = ConfigFactory.load();
  File file = new File(configfile);
  assert (file.exists());
  Config testResourceConfig = ConfigFactory.parseFileAnySyntax(file, ConfigParseOptions.defaults().setAllowMissing(false));

  Config typesafe  = testResourceConfig.withFallback(reference).resolve();

  StreamsConfiguration streamsConfiguration = StreamsConfigurator.detectConfiguration(typesafe);
  MoreoverConfiguration config = new ComponentConfigurator<>(MoreoverConfiguration.class).detectConfiguration(typesafe, "rss");
  MoreoverProvider provider = new MoreoverProvider(config);

  ObjectMapper mapper = StreamsJacksonMapper.getInstance();

  PrintStream outStream = new PrintStream(new BufferedOutputStream(new FileOutputStream(outfile)));
  provider.prepare(config);
  provider.startStream();
  do {
    Uninterruptibles.sleepUninterruptibly(streamsConfiguration.getBatchFrequencyMs(), TimeUnit.MILLISECONDS);
    for (StreamsDatum datum : provider.readCurrent()) {
      String json;
      try {
        json = mapper.writeValueAsString(datum.getDocument());
        outStream.println(json);
      } catch (JsonProcessingException ex) {
        System.err.println(ex.getMessage());
      }
    }
  }
  while ( provider.isRunning() );
  provider.cleanUp();
  outStream.flush();
}
Example 3
Source File: GPlusUserActivityProvider.java    From streams with Apache License 2.0 5 votes vote down vote up 
/**
 * Retrieve recent activity from a list of accounts.
 * @param args args
 * @throws Exception Exception
 */
public static void main(String[] args) throws Exception {

  Preconditions.checkArgument(args.length >= 2);

  String configfile = args[0];
  String outfile = args[1];

  File file = new File(configfile);
  assert (file.exists());

  Config conf = ConfigFactory.parseFileAnySyntax(file, ConfigParseOptions.defaults().setAllowMissing(false));
  StreamsConfigurator.addConfig(conf);

  StreamsConfiguration streamsConfiguration = StreamsConfigurator.detectConfiguration();
  GPlusUserActivityProviderConfiguration config = new ComponentConfigurator<>(GPlusUserActivityProviderConfiguration.class).detectConfiguration();
  GPlusUserActivityProvider provider = new GPlusUserActivityProvider(config);

  Gson gson = new Gson();

  PrintStream outStream = new PrintStream(new BufferedOutputStream(new FileOutputStream(outfile)));
  provider.prepare(config);
  provider.startStream();
  do {
    Uninterruptibles.sleepUninterruptibly(streamsConfiguration.getBatchFrequencyMs(), TimeUnit.MILLISECONDS);
    for (StreamsDatum datum : provider.readCurrent()) {
      String json;
      if (datum.getDocument() instanceof String) {
        json = (String) datum.getDocument();
      } else {
        json = gson.toJson(datum.getDocument());
      }
      outStream.println(json);
    }
  }
  while ( provider.isRunning());
  provider.cleanUp();
  outStream.flush();
}
Example 4
Source File: GPlusUserDataProvider.java    From streams with Apache License 2.0 5 votes vote down vote up 
/**
 * Retrieve current profile status for a list of accounts.
 * @param args args
 * @throws Exception Exception
 */
public static void main(String[] args) throws Exception {

  MatcherAssert.assertThat(args.length, Matchers.greaterThanOrEqualTo(2));

  String configfile = args[0];
  String outfile = args[1];

  File file = new File(configfile);
  assert (file.exists());

  Config conf = ConfigFactory.parseFileAnySyntax(file, ConfigParseOptions.defaults().setAllowMissing(false));
  StreamsConfigurator.addConfig(conf);

  StreamsConfiguration streamsConfiguration = StreamsConfigurator.detectConfiguration();
  GPlusUserDataProviderConfiguration config = new ComponentConfigurator<>(GPlusUserDataProviderConfiguration.class).detectConfiguration();
  GPlusUserDataProvider provider = new GPlusUserDataProvider(config);

  Gson gson = new Gson();

  PrintStream outStream = new PrintStream(new BufferedOutputStream(new FileOutputStream(outfile)));
  provider.prepare(config);
  provider.startStream();
  do {
    Uninterruptibles.sleepUninterruptibly(streamsConfiguration.getBatchFrequencyMs(), TimeUnit.MILLISECONDS);
    for (StreamsDatum datum : provider.readCurrent()) {
      String json;
      if (datum.getDocument() instanceof String) {
        json = (String) datum.getDocument();
      } else {
        json = gson.toJson(datum.getDocument());
      }
      outStream.println(json);
    }
  }
  while ( provider.isRunning());
  provider.cleanUp();
  outStream.flush();
}
Example 5
Source File: FacebookPageFeedProvider.java    From streams with Apache License 2.0 5 votes vote down vote up 
/**
 * Run FacebookPageFeedProvider from command line.
 * @param args configfile outfile
 * @throws Exception Exception
 */
public static void main(String[] args) throws Exception {

  Preconditions.checkArgument(args.length >= 2);

  String configfile = args[0];
  String outfile = args[1];

  Config reference = ConfigFactory.load();
  File confFile = new File(configfile);
  assert (confFile.exists());

  Config conf = ConfigFactory.parseFileAnySyntax(confFile, ConfigParseOptions.defaults().setAllowMissing(false));
  StreamsConfigurator.addConfig(conf);

  StreamsConfiguration streamsConfiguration = StreamsConfigurator.detectConfiguration();
  FacebookPageFeedProviderConfiguration config = new ComponentConfigurator<>(FacebookPageFeedProviderConfiguration.class).detectConfiguration();
  FacebookPageFeedProvider provider = new FacebookPageFeedProvider(config);

  PrintStream outStream = new PrintStream(new BufferedOutputStream(new FileOutputStream(outfile)));
  provider.prepare(config);
  provider.startStream();
  do {
    Uninterruptibles.sleepUninterruptibly(streamsConfiguration.getBatchFrequencyMs(), TimeUnit.MILLISECONDS);
    for (StreamsDatum datum : provider.readCurrent()) {
      String json;
      try {
        json = MAPPER.writeValueAsString(datum.getDocument());
        outStream.println(json);
      } catch (JsonProcessingException ex) {
        System.err.println(ex.getMessage());
      }
    }
  }
  while ( provider.isRunning());
  provider.cleanUp();
  outStream.flush();
}
Example 6
Source File: FacebookPageProvider.java    From streams with Apache License 2.0 5 votes vote down vote up 
/**
 * Run FacebookPageProvider from command line.
 * @param args configfile outfile
 * @throws Exception Exception
 */
public static void main(String[] args) throws Exception {

  Preconditions.checkArgument(args.length >= 2);

  String configfile = args[0];
  String outfile = args[1];

  File confFile = new File(configfile);
  assert (confFile.exists());

  Config conf = ConfigFactory.parseFileAnySyntax(confFile, ConfigParseOptions.defaults().setAllowMissing(false));
  StreamsConfigurator.addConfig(conf);

  StreamsConfiguration streamsConfiguration = StreamsConfigurator.detectConfiguration();
  FacebookPageProviderConfiguration config = new ComponentConfigurator<>(FacebookPageProviderConfiguration.class).detectConfiguration();
  FacebookPageProvider provider = new FacebookPageProvider(config);

  PrintStream outStream = new PrintStream(new BufferedOutputStream(new FileOutputStream(outfile)));
  provider.prepare(config);
  provider.startStream();
  do {
    Uninterruptibles.sleepUninterruptibly(streamsConfiguration.getBatchFrequencyMs(), TimeUnit.MILLISECONDS);
    for (StreamsDatum datum : provider.readCurrent()) {
      String json;
      try {
        json = MAPPER.writeValueAsString(datum.getDocument());
        outStream.println(json);
      } catch (JsonProcessingException ex) {
        System.err.println(ex.getMessage());
      }
    }
  }
  while ( provider.isRunning());
  provider.cleanUp();
  outStream.flush();
}
Example 7
Source File: InstagramRecentMediaProvider.java    From streams with Apache License 2.0 5 votes vote down vote up 
/**
 * To use from command line:
 *
 * <p></p>
 * Supply (at least) the following required configuration in application.conf:
 *
 * <p></p>
 * instagram.clientKey
 * instagram.usersInfo.authorizedTokens
 * instagram.usersInfo.users
 *
 * <p></p>
 * Launch using:
 *
 * <p></p>
 * mvn exec:java \
 * -Dexec.mainClass=org.apache.streams.instagram.provider.recentmedia.InstagramRecentMediaProvider \
 * -Dexec.args="application.conf media.json.txt"
 *
 * @param args args
 * @throws Exception Exception
 */
public static void main(String[] args) throws Exception {

  Preconditions.checkArgument(args.length >= 2);

  String configfile = args[0];
  String outfile = args[1];

  Config reference = ConfigFactory.load();
  File file = new File(configfile);
  assert (file.exists());

  Config conf = ConfigFactory.parseFileAnySyntax(file, ConfigParseOptions.defaults().setAllowMissing(false));
  StreamsConfigurator.addConfig(conf);

  StreamsConfiguration streamsConfiguration = StreamsConfigurator.detectConfiguration();
  InstagramRecentMediaProviderConfiguration config = new ComponentConfigurator<>(InstagramRecentMediaProviderConfiguration.class).detectConfiguration();
  InstagramRecentMediaProvider provider = new InstagramRecentMediaProvider(config);

  PrintStream outStream = new PrintStream(new BufferedOutputStream(new FileOutputStream(outfile)));
  provider.prepare(config);
  provider.startStream();
  do {
    Uninterruptibles.sleepUninterruptibly(streamsConfiguration.getBatchFrequencyMs(), TimeUnit.MILLISECONDS);
    Iterator<StreamsDatum> iterator = provider.readCurrent().iterator();
    while (iterator.hasNext()) {
      StreamsDatum datum = iterator.next();
      String json;
      try {
        json = MAPPER.writeValueAsString(datum.getDocument());
        outStream.println(json);
      } catch (JsonProcessingException ex) {
        System.err.println(ex.getMessage());
      }
    }
  }
  while ( provider.isRunning());
  provider.cleanUp();
  outStream.flush();
}
Example 8
Source File: InstagramUserInfoProvider.java    From streams with Apache License 2.0 5 votes vote down vote up 
/**
 * To use from command line:
 *
 * <p></p>
 * Supply (at least) the following required configuration in application.conf:
 *
 * <p></p>
 * instagram.clientKey
 * instagram.usersInfo.authorizedTokens
 * instagram.usersInfo.users
 *
 * <p></p>
 * Launch using:
 *
 * <p></p>
 * mvn exec:java \
 * -Dexec.mainClass=org.apache.streams.instagram.provider.userinfo.InstagramUserInfoProvider \
 * -Dexec.args="application.conf userinfo.json"
 *
 * @param args args
 * @throws Exception Exception
 */
public static void main(String[] args) throws Exception {

  Preconditions.checkArgument(args.length >= 2);

  String configfile = args[0];
  String outfile = args[1];

  Config reference = ConfigFactory.load();
  File file = new File(configfile);
  assert (file.exists());

  Config conf = ConfigFactory.parseFileAnySyntax(file, ConfigParseOptions.defaults().setAllowMissing(false));
  StreamsConfigurator.addConfig(conf);

  StreamsConfiguration streamsConfiguration = StreamsConfigurator.detectConfiguration();
  InstagramUserInfoProviderConfiguration config = new ComponentConfigurator<>(InstagramUserInfoProviderConfiguration.class).detectConfiguration();
  InstagramUserInfoProvider provider = new InstagramUserInfoProvider(config);

  PrintStream outStream = new PrintStream(new BufferedOutputStream(new FileOutputStream(outfile)));
  provider.prepare(config);
  provider.startStream();
  do {
    Uninterruptibles.sleepUninterruptibly(streamsConfiguration.getBatchFrequencyMs(), TimeUnit.MILLISECONDS);
    Iterator<StreamsDatum> iterator = provider.readCurrent().iterator();
    while (iterator.hasNext()) {
      StreamsDatum datum = iterator.next();
      String json;
      try {
        json = MAPPER.writeValueAsString(datum.getDocument());
        outStream.println(json);
      } catch (JsonProcessingException ex) {
        System.err.println(ex.getMessage());
      }
    }
  }
  while ( provider.isRunning());
  provider.cleanUp();
  outStream.flush();
}
Example 9
Source File: InstagramCommentsProvider.java    From streams with Apache License 2.0 5 votes vote down vote up 
/**
 * To use from command line:
 *
 * <p></p>
 * Supply (at least) the following required configuration in application.conf:
 *
 * <p></p>
 * instagram.clientKey
 * instagram.usersInfo.authorizedTokens
 * instagram.usersInfo.users
 *
 * <p></p>
 * Launch using:
 *
 * <p></p>
 * mvn exec:java \
 * -Dexec.mainClass=org.apache.streams.instagram.provider.recentmedia.InstagramCommentsProvider \
 * -Dexec.args="application.conf comments.json.txt"
 *
 * @param args args
 * @throws Exception Exception
 */
public static void main(String[] args) throws Exception {

  Preconditions.checkArgument(args.length >= 2);

  String configfile = args[0];
  String outfile = args[1];

  Config reference = ConfigFactory.load();
  File file = new File(configfile);
  assert (file.exists());

  Config conf = ConfigFactory.parseFileAnySyntax(file, ConfigParseOptions.defaults().setAllowMissing(false));
  StreamsConfigurator.addConfig(conf);

  StreamsConfiguration streamsConfiguration = StreamsConfigurator.detectConfiguration();
  InstagramCommentsProviderConfiguration config = new ComponentConfigurator<>(InstagramCommentsProviderConfiguration.class).detectConfiguration();
  InstagramCommentsProvider provider = new InstagramCommentsProvider(config);

  PrintStream outStream = new PrintStream(new BufferedOutputStream(new FileOutputStream(outfile)));
  provider.prepare(config);
  provider.startStream();
  do {
    Uninterruptibles.sleepUninterruptibly(streamsConfiguration.getBatchFrequencyMs(), TimeUnit.MILLISECONDS);
    Iterator<StreamsDatum> iterator = provider.readCurrent().iterator();
    while (iterator.hasNext()) {
      StreamsDatum datum = iterator.next();
      String json;
      try {
        json = MAPPER.writeValueAsString(datum.getDocument());
        outStream.println(json);
      } catch (JsonProcessingException ex) {
        System.err.println(ex.getMessage());
      }
    }
  }
  while ( provider.isRunning());
  provider.cleanUp();
  outStream.flush();
}
Example 10
Source File: SysomosProvider.java    From streams with Apache License 2.0 5 votes vote down vote up 
/**
 * To use from command line:
 * <p></p>
 * Supply configuration similar to src/test/resources/rss.conf
 * <p></p>
 * Launch using:
 * <p></p>
 * mvn exec:java -Dexec.mainClass=org.apache.streams.rss.provider.RssStreamProvider -Dexec.args="rss.conf articles.json"
 * @param args args
 * @throws Exception Exception
 */
public static void main(String[] args) throws Exception {

  Preconditions.checkArgument(args.length >= 2);

  String configfile = args[0];
  String outfile = args[1];

  Config reference = ConfigFactory.load();
  File file = new File(configfile);
  assert (file.exists());
  Config testResourceConfig = ConfigFactory.parseFileAnySyntax(file, ConfigParseOptions.defaults().setAllowMissing(false));

  Config typesafe = testResourceConfig.withFallback(reference).resolve();

  StreamsConfiguration streamsConfiguration = StreamsConfigurator.detectConfiguration(typesafe);
  SysomosConfiguration config = new ComponentConfigurator<>(SysomosConfiguration.class).detectConfiguration(typesafe, "rss");
  SysomosProvider provider = new SysomosProvider(config);

  ObjectMapper mapper = StreamsJacksonMapper.getInstance();

  PrintStream outStream = new PrintStream(new BufferedOutputStream(new FileOutputStream(outfile)));
  provider.prepare(config);
  provider.startStream();
  do {
    Uninterruptibles.sleepUninterruptibly(streamsConfiguration.getBatchFrequencyMs(), TimeUnit.MILLISECONDS);
    for (StreamsDatum datum : provider.readCurrent()) {
      String json;
      try {
        json = mapper.writeValueAsString(datum.getDocument());
        outStream.println(json);
      } catch (JsonProcessingException ex) {
        System.err.println(ex.getMessage());
      }
    }
  }
  while (provider.isRunning());
  provider.cleanUp();
  outStream.flush();
}
Example 11
Source File: SprinklrBootstrapIT.java    From streams with Apache License 2.0 5 votes vote down vote up 
@BeforeClass(alwaysRun = true)
public void setup() throws Exception {
  File conf = new File(configfile);
  Assert.assertTrue(conf.exists());
  Assert.assertTrue(conf.canRead());
  Assert.assertTrue(conf.isFile());
  Config parsedConfig = ConfigFactory.parseFileAnySyntax(conf);
  StreamsConfigurator.addConfig(parsedConfig);
  config = new ComponentConfigurator<>(SprinklrConfiguration.class).detectConfiguration();
  testsconfig = StreamsConfigurator.getConfig().getConfig("org.apache.streams.sprinklr.config.SprinklrConfiguration");
}
Example 12
Source File: RssStreamProvider.java    From streams with Apache License 2.0 5 votes vote down vote up 
/**
 * To use from command line:
 *
 * <p></p>
 * Supply configuration similar to src/test/resources/rss.conf
 *
 * <p></p>
 * Launch using:
 *
 * <p></p>
 * mvn exec:java -Dexec.mainClass=org.apache.streams.rss.provider.RssStreamProvider -Dexec.args="rss.conf articles.json"
 * @param args args
 * @throws Exception Exception
 */
public static void main(String[] args) throws Exception {

  Preconditions.checkArgument(args.length >= 2);

  String configfile = args[0];
  String outfile = args[1];

  Config reference = ConfigFactory.load();
  File file = new File(configfile);
  assert (file.exists());
  Config testResourceConfig = ConfigFactory.parseFileAnySyntax(file, ConfigParseOptions.defaults().setAllowMissing(false));

  Config typesafe  = testResourceConfig.withFallback(reference).resolve();

  StreamsConfiguration streamsConfiguration = StreamsConfigurator.detectConfiguration(typesafe);
  RssStreamConfiguration config = new ComponentConfigurator<>(RssStreamConfiguration.class).detectConfiguration(typesafe, "rss");
  RssStreamProvider provider = new RssStreamProvider(config);

  ObjectMapper mapper = StreamsJacksonMapper.getInstance();

  PrintStream outStream = new PrintStream(new BufferedOutputStream(new FileOutputStream(outfile)));
  provider.prepare(config);
  provider.startStream();
  do {
    Uninterruptibles.sleepUninterruptibly(streamsConfiguration.getBatchFrequencyMs(), TimeUnit.MILLISECONDS);
    for (StreamsDatum datum : provider.readCurrent()) {
      String json;
      try {
        json = mapper.writeValueAsString(datum.getDocument());
        outStream.println(json);
      } catch (JsonProcessingException ex) {
        System.err.println(ex.getMessage());
      }
    }
  }
  while ( provider.isRunning());
  provider.cleanUp();
  outStream.flush();
}
Example 13
Source File: TwitterFollowingProvider.java    From streams with Apache License 2.0 5 votes vote down vote up 
/**
 * To use from command line:
 *
 * <p></p>
 * Supply (at least) the following required configuration in application.conf:
 *
 * <p></p>
 * twitter.oauth.consumerKey
 * twitter.oauth.consumerSecret
 * twitter.oauth.accessToken
 * twitter.oauth.accessTokenSecret
 * twitter.info
 *
 * <p></p>
 * Launch using:
 *
 * <p></p>
 * mvn exec:java -Dexec.mainClass=org.apache.streams.twitter.provider.TwitterFollowingProvider -Dexec.args="application.conf tweets.json"
 *
 * @param args args
 * @throws Exception Exception
 */
public static void main(String[] args) throws Exception {

  Preconditions.checkArgument(args.length >= 2);

  String configfile = args[0];
  String outfile = args[1];

  File file = new File(configfile);
  assert (file.exists());

  Config configFile = ConfigFactory.parseFileAnySyntax(file, ConfigParseOptions.defaults());
  StreamsConfigurator.addConfig(configFile);

  PrintStream outStream = new PrintStream(new BufferedOutputStream(new FileOutputStream(outfile)));
  ObjectMapper mapper = StreamsJacksonMapper.getInstance(new StreamsJacksonMapperConfiguration().withDateFormats(Stream.of(TwitterDateTimeFormat.TWITTER_FORMAT).collect(Collectors.toList())));

  TwitterFollowingConfiguration config = new ComponentConfigurator<>(TwitterFollowingConfiguration.class).detectConfiguration();
  TwitterFollowingProvider provider = new TwitterFollowingProvider(config);

  Iterator<Follow> results = provider.call();

  results.forEachRemaining(d -> {
    try {
      outStream.println(mapper.writeValueAsString(d));
    } catch( Exception e ) {
      LOGGER.warn("Exception", e);
    }
  });

  outStream.flush();
}
Example 14
Source File: TwitterUserInformationProvider.java    From streams with Apache License 2.0 5 votes vote down vote up 
/**
 * To use from command line:
 *
 * <p></p>
 * Supply (at least) the following required configuration in application.conf:
 *
 * <p></p>
 * twitter.oauth.consumerKey
 * twitter.oauth.consumerSecret
 * twitter.oauth.accessToken
 * twitter.oauth.accessTokenSecret
 * twitter.info
 *
 * <p></p>
 * Launch using:
 *
 * <p></p>
 * mvn exec:java -Dexec.mainClass=org.apache.streams.twitter.provider.TwitterUserInformationProvider -Dexec.args="application.conf tweets.json"
 *
 * @param args args
 * @throws Exception Exception
 */
public static void main(String[] args) throws Exception {

  Preconditions.checkArgument(args.length >= 2);

  String configfile = args[0];
  String outfile = args[1];

  File file = new File(configfile);
  assert (file.exists());

  Config testResourceConfig = ConfigFactory.parseFileAnySyntax(file, ConfigParseOptions.defaults().setAllowMissing(false));
  StreamsConfigurator.addConfig(testResourceConfig);

  StreamsConfiguration streamsConfiguration = StreamsConfigurator.detectConfiguration();
  TwitterUserInformationConfiguration config = new ComponentConfigurator<>(TwitterUserInformationConfiguration.class).detectConfiguration();
  TwitterUserInformationProvider provider = new TwitterUserInformationProvider(config);

  PrintStream outStream = new PrintStream(new BufferedOutputStream(new FileOutputStream(outfile)));
  provider.prepare(config);
  provider.startStream();
  do {
    Uninterruptibles.sleepUninterruptibly(streamsConfiguration.getBatchFrequencyMs(), TimeUnit.MILLISECONDS);
    for (StreamsDatum datum : provider.readCurrent()) {
      String json;
      try {
        json = MAPPER.writeValueAsString(datum.getDocument());
        outStream.println(json);
      } catch (JsonProcessingException ex) {
        System.err.println(ex.getMessage());
      }
    }
  }
  while ( provider.isRunning());
  provider.cleanUp();
  outStream.flush();
}
Example 15
Source File: ThirtyDaySearchProvider.java    From streams with Apache License 2.0 5 votes vote down vote up 
/**
 * To use from command line:
 *
 * <p></p>
 * Supply (at least) the following required configuration in application.conf:
 *
 * <p></p>
 * twitter.oauth.consumerKey
 * twitter.oauth.consumerSecret
 * twitter.oauth.accessToken
 * twitter.oauth.accessTokenSecret
 * twitter.info
 *
 * <p></p>
 * Launch using:
 *
 * <p></p>
 * mvn exec:java -Dexec.mainClass=org.apache.streams.twitter.provider.TwitterTimelineProvider -Dexec.args="application.conf tweets.json"
 *
 * @param args args
 * @throws Exception Exception
 */
public static void main(String[] args) throws Exception {

  Preconditions.checkArgument(args.length >= 2);

  String configfile = args[0];
  String outfile = args[1];

  Config reference = ConfigFactory.load();
  File file = new File(configfile);
  assert (file.exists());
  Config testResourceConfig = ConfigFactory.parseFileAnySyntax(file, ConfigParseOptions.defaults().setAllowMissing(false));

  Config typesafe  = testResourceConfig.withFallback(reference).resolve();

  StreamsConfiguration streamsConfiguration = StreamsConfigurator.detectConfiguration(typesafe);
  ThirtyDaySearchProviderConfiguration config = new ComponentConfigurator<>(ThirtyDaySearchProviderConfiguration.class).detectConfiguration();
  ThirtyDaySearchProvider provider = new ThirtyDaySearchProvider(config);

  PrintStream outStream = new PrintStream(new BufferedOutputStream(new FileOutputStream(outfile)));
  ObjectMapper mapper = StreamsJacksonMapper.getInstance(new StreamsJacksonMapperConfiguration().withDateFormats(Stream.of(TwitterDateTimeFormat.TWITTER_FORMAT).collect(Collectors.toList())));

  Iterator<Tweet> results = provider.call();

  results.forEachRemaining(d -> {
    try {
      outStream.println(mapper.writeValueAsString(d));
    } catch( Exception e ) {
      LOGGER.warn("Exception", e);
    }
  });

  outStream.flush();

}
Example 16
Source File: SevenDaySearchProvider.java    From streams with Apache License 2.0 5 votes vote down vote up 
/**
 * To use from command line:
 *
 * <p></p>
 * Supply (at least) the following required configuration in application.conf:
 *
 * <p></p>
 * twitter.oauth.consumerKey
 * twitter.oauth.consumerSecret
 * twitter.oauth.accessToken
 * twitter.oauth.accessTokenSecret
 * twitter.info
 *
 * <p></p>
 * Launch using:
 *
 * <p></p>
 * mvn exec:java -Dexec.mainClass=org.apache.streams.twitter.provider.TwitterTimelineProvider -Dexec.args="application.conf tweets.json"
 *
 * @param args args
 * @throws Exception Exception
 */
public static void main(String[] args) throws Exception {

  Preconditions.checkArgument(args.length >= 2);

  String configfile = args[0];
  String outfile = args[1];

  File file = new File(configfile);
  assert (file.exists());

  Config testResourceConfig = ConfigFactory.parseFileAnySyntax(file, ConfigParseOptions.defaults().setAllowMissing(false));
  StreamsConfigurator.addConfig(testResourceConfig);

  StreamsConfiguration streamsConfiguration = StreamsConfigurator.detectConfiguration();
  SevenDaySearchProviderConfiguration config = new ComponentConfigurator<>(SevenDaySearchProviderConfiguration.class).detectConfiguration();
  SevenDaySearchProvider provider = new SevenDaySearchProvider(config);

  PrintStream outStream = new PrintStream(new BufferedOutputStream(new FileOutputStream(outfile)));
  ObjectMapper mapper = StreamsJacksonMapper.getInstance(new StreamsJacksonMapperConfiguration().withDateFormats(Stream.of(TwitterDateTimeFormat.TWITTER_FORMAT).collect(Collectors.toList())));

  Iterator<Tweet> results = provider.call();

  results.forEachRemaining(d -> {
    try {
      outStream.println(mapper.writeValueAsString(d));
    } catch( Exception e ) {
      LOGGER.warn("Exception", e);
    }
  });

  outStream.flush();

}
Example 17
Source File: TwitterTimelineProvider.java    From streams with Apache License 2.0 5 votes vote down vote up 
/**
 * To use from command line:
 *
 * <p></p>
 * Supply (at least) the following required configuration in application.conf:
 *
 * <p></p>
 * twitter.oauth.consumerKey
 * twitter.oauth.consumerSecret
 * twitter.oauth.accessToken
 * twitter.oauth.accessTokenSecret
 * twitter.info
 *
 * <p></p>
 * Launch using:
 *
 * <p></p>
 * mvn exec:java -Dexec.mainClass=org.apache.streams.twitter.provider.TwitterTimelineProvider -Dexec.args="application.conf tweets.json"
 *
 * @param args args
 * @throws Exception Exception
 */
public static void main(String[] args) throws Exception {

  Preconditions.checkArgument(args.length >= 2);

  String configfile = args[0];
  String outfile = args[1];

  File file = new File(configfile);
  assert (file.exists());

  Config testResourceConfig = ConfigFactory.parseFileAnySyntax(file, ConfigParseOptions.defaults().setAllowMissing(false));
  StreamsConfigurator.addConfig(testResourceConfig);

  StreamsConfiguration streamsConfiguration = StreamsConfigurator.detectConfiguration();
  TwitterTimelineProviderConfiguration config = new ComponentConfigurator<>(TwitterTimelineProviderConfiguration.class).detectConfiguration();
  TwitterTimelineProvider provider = new TwitterTimelineProvider(config);

  ObjectMapper mapper = new StreamsJacksonMapper(Stream.of(TwitterDateTimeFormat.TWITTER_FORMAT).collect(Collectors.toList()));

  PrintStream outStream = new PrintStream(new BufferedOutputStream(new FileOutputStream(outfile)));
  provider.prepare(config);
  provider.startStream();
  do {
    Uninterruptibles.sleepUninterruptibly(streamsConfiguration.getBatchFrequencyMs(), TimeUnit.MILLISECONDS);
    for (StreamsDatum datum : provider.readCurrent()) {
      String json;
      try {
        json = mapper.writeValueAsString(datum.getDocument());
        outStream.println(json);
      } catch (JsonProcessingException ex) {
        System.err.println(ex.getMessage());
      }
    }
  }
  while ( provider.isRunning() );
  provider.cleanUp();
  outStream.flush();
}
Example 18
Source File: YoutubeUserActivityProvider.java    From streams with Apache License 2.0 4 votes vote down vote up 
/**
 * To use from command line:
 * <p>
 * <p></p>
 * Supply (at least) the following required configuration in application.conf:
 * <p>
 * <p></p>
 * youtube.oauth.pathToP12KeyFile
 * youtube.oauth.serviceAccountEmailAddress
 * youtube.apiKey
 * youtube.youtubeUsers
 * <p>
 * <p></p>
 * Launch using:
 * <p>
 * <p></p>
 * mvn exec:java -Dexec.mainClass=org.apache.streams.youtube.provider.YoutubeUserActivityProvider -Dexec.args="application.conf tweets.json"
 *
 * @param args args
 * @throws Exception Exception
 */
public static void main(String[] args) throws Exception {

  Preconditions.checkArgument(args.length >= 2);

  String configfile = args[0];
  String outfile = args[1];

  File file = new File(configfile);
  assert (file.exists());

  Config conf = ConfigFactory.parseFileAnySyntax(file, ConfigParseOptions.defaults().setAllowMissing(false));
  StreamsConfigurator.addConfig(conf);

  StreamsConfiguration streamsConfiguration = StreamsConfigurator.detectConfiguration();
  YoutubeUserActivityProviderConfiguration config = new ComponentConfigurator<>(YoutubeUserActivityProviderConfiguration.class).detectConfiguration();
  YoutubeUserActivityProvider provider = new YoutubeUserActivityProvider(config);

  ObjectMapper mapper = StreamsJacksonMapper.getInstance();

  PrintStream outStream = new PrintStream(new BufferedOutputStream(new FileOutputStream(outfile)));
  provider.prepare(config);
  provider.startStream();
  do {
    Uninterruptibles.sleepUninterruptibly(streamsConfiguration.getBatchFrequencyMs(), TimeUnit.MILLISECONDS);
    Iterator<StreamsDatum> iterator = provider.readCurrent().iterator();
    while (iterator.hasNext()) {
      StreamsDatum datum = iterator.next();
      String json;
      try {
        if (datum.getDocument() instanceof String) {
          json = (String) datum.getDocument();
        } else {
          json = mapper.writeValueAsString(datum.getDocument());
        }
        outStream.println(json);
      } catch (JsonProcessingException ex) {
        System.err.println(ex.getMessage());
      }
    }
  }
  while (provider.isRunning());
  provider.cleanUp();
  outStream.flush();
}
Example 19
Source File: YoutubeChannelProvider.java    From streams with Apache License 2.0 4 votes vote down vote up 
/**
 * To use from command line:
 * <p></p>
 * Supply (at least) the following required configuration in application.conf:
 * <p></p>
 * youtube.oauth.pathToP12KeyFile
 * youtube.oauth.serviceAccountEmailAddress
 * youtube.apiKey
 * youtube.youtubeUsers
 * <p></p>
 * Launch using:
 * <p></p>
 * mvn exec:java -Dexec.mainClass=org.apache.streams.youtube.provider.YoutubeUserActivityProvider -Dexec.args="application.conf tweets.json"
 * @param args args
 * @throws Exception Exception
 */
public static void main(String[] args) throws Exception {

  Preconditions.checkArgument(args.length >= 2);

  String configfile = args[0];
  String outfile = args[1];

  File file = new File(configfile);
  assert (file.exists());

  Config conf = ConfigFactory.parseFileAnySyntax(file, ConfigParseOptions.defaults().setAllowMissing(false));
  StreamsConfigurator.addConfig(conf);

  StreamsConfiguration streamsConfiguration = StreamsConfigurator.detectConfiguration();
  YoutubeChannelProviderConfiguration config = new ComponentConfigurator<>(YoutubeChannelProviderConfiguration.class).detectConfiguration();
  YoutubeUserActivityProvider provider = new YoutubeUserActivityProvider(config);

  ObjectMapper mapper = StreamsJacksonMapper.getInstance();

  PrintStream outStream = new PrintStream(new BufferedOutputStream(new FileOutputStream(outfile)));
  provider.prepare(config);
  provider.startStream();
  do {
    Uninterruptibles.sleepUninterruptibly(streamsConfiguration.getBatchFrequencyMs(), TimeUnit.MILLISECONDS);
    for (StreamsDatum datum : provider.readCurrent()) {
      String json;
      try {
        if (datum.getDocument() instanceof String) {
          json = (String) datum.getDocument();
        } else {
          json = mapper.writeValueAsString(datum.getDocument());
        }
        outStream.println(json);
      } catch (JsonProcessingException ex) {
        System.err.println(ex.getMessage());
      }
    }
  }
  while (provider.isRunning());
  provider.cleanUp();
  outStream.flush();
}
Example 20
Source File: FileBasedConfigSupplier.java    From ditto with Eclipse Public License 2.0 2 votes vote down vote up 
/**
 * Returns an instance of {@code FileBasedConfigSupplier} that provides a {@link com.typesafe.config.Config} based
 * on the file denoted by system environment variable {@value #HOSTING_ENV_FILE_LOCATION_ENV_VARIABLE_NAME}.
 *
 * @return the instance.
 * @throws DittoConfigError if the value of the system environment variable
 * {@value #HOSTING_ENV_FILE_LOCATION_ENV_VARIABLE_NAME} was either not set at all or did not denote an existing
 * file.
 */
static FileBasedConfigSupplier forConfiguredConfigFile() {
    final Config initialConfig = ConfigFactory.parseFileAnySyntax(getConfigFile(getConfigFileLocation()));
    return new FileBasedConfigSupplier(initialConfig);
}