com.twitter.hbc.core.endpoint.StatusesFilterEndpoint Java Examples

@Test
public void testStream_withTermFilter() {
    Pipeline pipeline = Pipeline.create();
    List<String> terms = new ArrayList<String>(Arrays.asList("BTC", "ETH"));
    final StreamSource<String> twitterTestStream = TwitterSources.stream(
            credentials, () -> new StatusesFilterEndpoint().trackTerms(terms));
    StreamStage<String> tweets = pipeline
            .readFrom(twitterTestStream)
            .withoutTimestamps()
            .map(rawJson -> Json.parse(rawJson)
                                .asObject()
                                .getString("text", null));

    tweets.writeTo(AssertionSinks.assertCollectedEventually(60,
            list -> assertGreaterOrEquals("Emits at least 20 tweets in 1 min.", list.size(), 20)));
    Job job = jet.newJob(pipeline);
    sleepAtLeastSeconds(5);
    try {
        job.join();
        fail("Job should have completed with an AssertionCompletedException, but completed normally");
    } catch (CompletionException e) {
        String errorMsg = e.getCause().getMessage();
        assertTrue("Job was expected to complete with AssertionCompletedException, but completed with: "
                + e.getCause(), errorMsg.contains(AssertionCompletedException.class.getName()));
    }
}
 

private  Client getTwitterClient(Properties props, BlockingQueue<String> messageQueue) {

        String clientName = props.getProperty("clientName");
        String consumerKey = props.getProperty("consumerKey");
        String consumerSecret = props.getProperty("consumerSecret");
        String token = props.getProperty("token");
        String tokenSecret = props.getProperty("tokenSecret");
        List<String> searchTerms = Arrays.asList(props.getProperty("searchTerms").split(","));

        Authentication authentication = new OAuth1(consumerKey,consumerSecret,token,tokenSecret);
        Hosts hosebirdHosts = new HttpHosts(Constants.STREAM_HOST);
        StatusesFilterEndpoint hosebirdEndpoint = new StatusesFilterEndpoint();

        hosebirdEndpoint.trackTerms(searchTerms);

        ClientBuilder clientBuilder = new ClientBuilder();
        clientBuilder.name(clientName)
                .hosts(hosebirdHosts)
                .authentication(authentication)
                .endpoint(hosebirdEndpoint)
                .processor(new StringDelimitedProcessor(messageQueue));

          return clientBuilder.build();

    }
 

private StreamingEndpoint getEndpoint(TwitterFireHoseConfig config) {
    List<Long> followings = config.getFollowings();
    List<String> terms = config.getTrackTerms();

    if (CollectionUtils.isEmpty(followings) && CollectionUtils.isEmpty(terms)) {
        return new SampleStatusesEndpoint().createEndpoint();
    } else {
        StatusesFilterEndpoint hosebirdEndpoint = new StatusesFilterEndpoint();

        if (CollectionUtils.isNotEmpty(followings)) {
           hosebirdEndpoint.followings(followings);
        }

        if (CollectionUtils.isNotEmpty(terms)) {
           hosebirdEndpoint.trackTerms(terms);
        }

        return hosebirdEndpoint;
    }
}
 

@Test
public void streamApiMockTest() {

    List<String> terms = new ArrayList<>(Arrays.asList("San Mateo", "Brno", "London", "Istanbul"));
    final StreamSource<String> twitterTestStream = TwitterSources.timestampedStream(getCredentials(),
            "http://" + server.getHostName() + ":" + server.getPort(),
            () -> new StatusesFilterEndpoint().trackTerms(terms));

    Pipeline pipeline = Pipeline.create();
    StreamStage<String> tweets = pipeline
            .readFrom(twitterTestStream)
            .withNativeTimestamps(0)
            .map(rawJson -> Json.parse(rawJson)
                                .asObject()
                                .getString("text", null));
    tweets.writeTo(AssertionSinks.assertCollectedEventually(10,
            list -> assertGreaterOrEquals("Emits at least 100 tweets in 1 min.",
                    list.size(), 100)));
    Job job = createJetMember().newJob(pipeline);

    sleepAtLeastSeconds(5);

    try {
        job.join();
        fail("Job should have completed with an AssertionCompletedException, but completed normally");
    } catch (CompletionException e) {
        String errorMsg = e.getCause().getMessage();
        assertTrue("Job was expected to complete with AssertionCompletedException, but completed with: "
                + e.getCause(), errorMsg.contains(AssertionCompletedException.class.getName()));
    }
}
 

@Test
public void testStream_userFilter() {
    Pipeline pipeline = Pipeline.create();
    List<Long> userIds = new ArrayList<Long>(
            Arrays.asList(612473L, 759251L, 1367531L, 34713362L, 51241574L, 87818409L));
    final StreamSource<String> twitterTestStream = TwitterSources.stream(credentials,
            () -> new StatusesFilterEndpoint().followings(userIds));
    StreamStage<String> tweets = pipeline
            .readFrom(twitterTestStream)
            .withoutTimestamps()
            .map(rawJson -> Json.parse(rawJson)
                                .asObject()
                                .getString("text", null));
    tweets.writeTo(AssertionSinks.assertCollectedEventually(60,
            list -> assertGreaterOrEquals("Emits at least 15 tweets in 1 min.",
                    list.size(), 15)));
    Job job = jet.newJob(pipeline);
    sleepAtLeastSeconds(5);
    try {
        job.join();
        fail("Job should have completed with an AssertionCompletedException, but completed normally");
    } catch (CompletionException e) {
        String errorMsg = e.getCause().getMessage();
        assertTrue("Job was expected to complete with AssertionCompletedException, but completed with: "
                + e.getCause(), errorMsg.contains(AssertionCompletedException.class.getName()));
    }
}
 

@Test
public void testTimestampedStream_termFilter() {
    Pipeline pipeline = Pipeline.create();
    List<String> terms = new ArrayList<String>(Arrays.asList("San Mateo", "Brno", "London", "Istanbul"));

    final StreamSource<String> twitterTestStream = TwitterSources.timestampedStream(
            credentials, () -> new StatusesFilterEndpoint().trackTerms(terms));
    StreamStage<String> tweets = pipeline
            .readFrom(twitterTestStream)
            .withNativeTimestamps(0)
            .map(rawJson -> Json.parse(rawJson)
                                .asObject()
                                .getString("text", null));
    tweets.writeTo(AssertionSinks.assertCollectedEventually(60,
            list -> assertGreaterOrEquals("Emits at least 20 tweets in 1 min.",
                    list.size(), 20)));
    Job job = jet.newJob(pipeline);
    sleepAtLeastSeconds(5);
    try {
        job.join();
        fail("Job should have completed with an AssertionCompletedException, but completed normally");
    } catch (CompletionException e) {
        String errorMsg = e.getCause().getMessage();
        assertTrue("Job was expected to complete with AssertionCompletedException, but completed with: "
                + e.getCause(), errorMsg.contains(AssertionCompletedException.class.getName()));
    }
}
 

@Test
public void testTimestampedStream_userFilter() {
    Pipeline pipeline = Pipeline.create();
    List<Long> userIds = new ArrayList<Long>(
            Arrays.asList(612473L, 759251L, 1367531L, 34713362L, 51241574L, 87818409L));
    final StreamSource<String> twitterTestStream = TwitterSources.timestampedStream(
            credentials, () -> new StatusesFilterEndpoint().followings(userIds));
    StreamStage<String> tweets = pipeline
            .readFrom(twitterTestStream)
            .withNativeTimestamps(1)
            .map(rawJson -> Json.parse(rawJson)
                                .asObject()
                                .getString("text", null));
    tweets.writeTo(AssertionSinks.assertCollectedEventually(60,
            list -> assertGreaterOrEquals("Emits at least 15 tweets in 1 min.",
                    list.size(), 15)));
    Job job = jet.newJob(pipeline);
    sleepAtLeastSeconds(5);
    try {
        job.join();
        fail("Job should have completed with an AssertionCompletedException, but completed normally");
    } catch (CompletionException e) {
        String errorMsg = e.getCause().getMessage();
        assertTrue("Job was expected to complete with AssertionCompletedException, but completed with: "
                + e.getCause(), errorMsg.contains(AssertionCompletedException.class.getName()));
    }
}
 

public TwitterStream(SentimentDetector sentimentDetector,
                     Gson gson,
                     Properties props) {
    this.sentimentDetector = sentimentDetector;
    this.gson = gson;

    msgQueue = new LinkedBlockingQueue<String>(100000);

    Hosts hosts = new HttpHosts(Constants.STREAM_HOST);
    StatusesFilterEndpoint endpoint = new StatusesFilterEndpoint();

    List<String> terms = Lists.newArrayList(
            props.getProperty("twitter_terms")
                    .split("\\s*,\\s*"));
    endpoint.trackTerms(terms);

    Authentication auth = new OAuth1(
            props.getProperty("twitter_consumer_key"),
            props.getProperty("twitter_consumer_secret"),
            props.getProperty("twitter_token"),
            props.getProperty("twitter_token_secret"));
    ClientBuilder builder = new ClientBuilder()
            .name("SmartCode-Client-01")
            .hosts(hosts)
            .authentication(auth)
            .endpoint(endpoint)
            .processor(new StringDelimitedProcessor(msgQueue));
    client = builder.build();
    client.connect();
}
 

public TwitterStream(Gson gson, Properties props, BlockingQueue<Map<String,Object>> imageQueue) {
    this.gson = gson;
    this.imageQueue = imageQueue;

    msgQueue = new LinkedBlockingQueue<String>(100000);

    Hosts hosts = new HttpHosts(Constants.STREAM_HOST);
    StatusesFilterEndpoint endpoint = new StatusesFilterEndpoint();

    List<String> terms = Lists.newArrayList(
            props.getProperty("twitter_terms")
                    .split("\\s*,\\s*"));
    endpoint.trackTerms(terms);

    Authentication auth = new OAuth1(
            props.getProperty("twitter_consumer_key"),
            props.getProperty("twitter_consumer_secret"),
            props.getProperty("twitter_token"),
            props.getProperty("twitter_token_secret"));
    ClientBuilder builder = new ClientBuilder()
            .name("SmartCode-Client-01")
            .hosts(hosts)
            .authentication(auth)
            .endpoint(endpoint)
            .processor(new StringDelimitedProcessor(msgQueue));
    client = builder.build();
    client.connect();
}
 

/**
 * Builds and returns the Pipeline which represents the actual computation.
 */
private static Pipeline buildPipeline() {
    Pipeline pipeline = Pipeline.create();

    List<String> allCoinMarkers = Stream.of(CoinType.values())
                                        .flatMap(ct -> ct.markers().stream())
                                        .collect(toList());
    StreamStage<String> tweets = pipeline
            .readFrom(TwitterSources.timestampedStream(
                    loadCredentials(),
                    () -> new StatusesFilterEndpoint()
                            .trackTerms(allCoinMarkers)))
            .withNativeTimestamps(SECONDS.toMillis(1));
    StreamStageWithKey<Entry<CoinType, Double>, CoinType> tweetsWithSentiment = tweets
            .map(rawTweet -> new JSONObject(rawTweet).getString("text"))
            .flatMap(CryptocurrencySentimentAnalysis::flatMapToRelevant)
            .mapUsingService(sentimentAnalyzerContext(), (analyzer, e1) ->
                    entry(e1.getKey(), analyzer.getSentimentScore(e1.getValue())))
            .filter(e -> !e.getValue().isInfinite() && !e.getValue().isNaN())
            .groupingKey(entryKey());

    AggregateOperation1<Entry<CoinType, Double>, ?, Tuple2<Double, Long>> avgAndCount =
            allOf(averagingDouble(Entry::getValue), counting());

    tweetsWithSentiment
            .window(sliding(HALF_MINUTE.durationMillis(), 200))
            .aggregate(avgAndCount)
            .map(windowResult -> entry(tuple2(windowResult.getKey(), HALF_MINUTE), windowResult.getValue()))
            .writeTo(map(MAP_NAME_JET_RESULTS));

    tweetsWithSentiment
            .window(sliding(FIVE_MINUTES.durationMillis(), 200))
            .aggregate(avgAndCount)
            .map(windowResult -> entry(tuple2(windowResult.getKey(), FIVE_MINUTES), windowResult.getValue()))
            .writeTo(map(MAP_NAME_JET_RESULTS));

    return pipeline;
}
 

@Override
protected void initializeConnection() {
    if (LOG.isInfoEnabled()) {
        LOG.info("Initializing Twitter Streaming API connection");
    }

    this.queue = new LinkedBlockingQueue(this.queueSize);
    StatusesFilterEndpoint endpoint = new StatusesFilterEndpoint(false)
            .locations(Arrays.asList(
                            new Location(
                                    // europa: -32.0 34.0 40.0 75.0
                                    new Location.Coordinate(-32.0, 34.0), // south west
                                    new Location.Coordinate(40.0, 75.0))
                     //       new Location(
                     //               // north america: -168.48633, 13.23995 -50.36133, 72.76406
                     //               new Location.Coordinate(-168.48633, 13.23995), // south west
                     //               new Location.Coordinate(-50.36133, 72.76406))
                    )
            );
    endpoint.stallWarnings(false);
    OAuth1 auth = this.authenticate();

    this.initializeClient(endpoint, auth);
    if (LOG.isInfoEnabled()) {
        LOG.info("Twitter Streaming API connection established successfully");
    }

}
 

public static void run(String consumerKey, String consumerSecret, String token, String secret) throws InterruptedException {
  BlockingQueue<String> queue = new LinkedBlockingQueue<String>(10000);
  StatusesFilterEndpoint endpoint = new StatusesFilterEndpoint();
  // add some track terms
  endpoint.trackTerms(Lists.newArrayList("twitterapi", "#yolo"));

  Authentication auth = new OAuth1(consumerKey, consumerSecret, token, secret);
  // Authentication auth = new BasicAuth(username, password);

  // Create a new BasicClient. By default gzip is enabled.
  Client client = new ClientBuilder()
          .hosts(Constants.STREAM_HOST)
          .endpoint(endpoint)
          .authentication(auth)
          .processor(new StringDelimitedProcessor(queue))
          .build();

  // Establish a connection
  client.connect();

  // Do whatever needs to be done with messages
  for (int msgRead = 0; msgRead < 1000; msgRead++) {
    String msg = queue.take();
    System.out.println(msg);
  }

  client.stop();

}
 

@PostConstruct
public void init() {
	BlockingQueue<String> queue = new LinkedBlockingQueue<>(100);
	StatusesFilterEndpoint endpoint = new StatusesFilterEndpoint();

	endpoint.trackTerms(ImmutableList.of("ExtJS", "#extjs", "Sencha", "#java",
			"java8", "java9", "#websocket", "#SpringFramework", "html5", "javascript",
			"#kotlin", "kotlin"));
	endpoint.languages(ImmutableList.of("en", "de"));

	String consumerKey = this.environment.getProperty("twitter4j.oauth.consumerKey");
	String consumerSecret = this.environment
			.getProperty("twitter4j.oauth.consumerSecret");
	String accessToken = this.environment.getProperty("twitter4j.oauth.accessToken");
	String accessTokenSecret = this.environment
			.getProperty("twitter4j.oauth.accessTokenSecret");

	Authentication auth = new OAuth1(consumerKey, consumerSecret, accessToken,
			accessTokenSecret);

	this.client = new ClientBuilder().hosts(Constants.STREAM_HOST).endpoint(endpoint)
			.authentication(auth).processor(new StringDelimitedProcessor(queue))
			.build();

	this.executorService = Executors.newSingleThreadExecutor();

	TwitterStatusListener statusListener = new TwitterStatusListener(
			this.messagingTemplate, this.lastTweets);
	this.t4jClient = new Twitter4jStatusClient(this.client, queue,
			ImmutableList.of(statusListener), this.executorService);

	this.t4jClient.connect();
	this.t4jClient.process();
}
 

public Stream<TweetHandler> stream() {
        String myKey = "sl2WbCf4UnIr08xvHVitHJ99r";
        String mySecret = "PE6yauvXjKLuvoQNXZAJo5C8N5U5piSFb3udwkoI76paK6KyqI";
        String myToken = "1098376471-p6iWfxCLtyMvMutTb010w1D1xZ3UyJhcC2kkBjN";
        String myAccess = "2o1uGcp4b2bFynOfu2cA1uz63n5aruV0RwNsUjRpjDBZS";

        out.println("Creating Twitter Stream");
        BlockingQueue<String> statusQueue = new LinkedBlockingQueue<>(1000);
        StatusesFilterEndpoint endpoint = new StatusesFilterEndpoint();
        endpoint.trackTerms(Lists.newArrayList("twitterapi", this.topic));
        endpoint.stallWarnings(false);
        Authentication twitterAuth = new OAuth1(myKey, mySecret, myToken, myAccess);

        BasicClient twitterClient = new ClientBuilder()
                .name("Twitter client")
                .hosts(Constants.STREAM_HOST)
                .endpoint(endpoint)
                .authentication(twitterAuth)
                .processor(new StringDelimitedProcessor(statusQueue))
                .build();

        twitterClient.connect();

        List<TweetHandler> list = new ArrayList();
        List<String> twitterList = new ArrayList();

        statusQueue.drainTo(twitterList);
        for(int i=0; i<numberOfTweets; i++) {
            String message;
            try {
                message = statusQueue.take();
                list.add(new TweetHandler(message));
            } catch (InterruptedException ex) {
                ex.printStackTrace();
            }
        }

//        for (int msgRead = 0; msgRead < this.numberOfTweets; msgRead++) {
//            try {
//                if (twitterClient.isDone()) {
//                  //  out.println(twitterClient.getExitEvent().getMessage());
//                    break;
//                }
//
//                String msg = statusQueue.poll(10, TimeUnit.SECONDS);
//                if (msg == null) {
//                    out.println("Waited 10 seconds - no message received");
//                } else {
//                    list.add(new TweetHandler(msg));
//                    out.println("Added message: " + msg.length());
//                }
//            } catch (InterruptedException ex) {
//                ex.printStackTrace();
//            }
//        }
        twitterClient.stop();
        out.printf("%d messages processed!\n", twitterClient.getStatsTracker().getNumMessages());

        return list.stream();
    }
 

public void run(String propFilePath) {
	loadFileProperties(propFilePath, DEFAULT_PROP_FILE_NAME);

	String consumerKey = System.getProperty(TWIT_CONSUMER_KEY);
	String consumerSecret = System.getProperty(TWIT_CONSUMER_SECRET);
	String token = System.getProperty(TWIT_TOKEN);
	String secret = System.getProperty(TWIT_SECRET);
	String streamName = System.getProperty(STREAM_NAME);
	String regionName = System.getProperty(REGION_NAME);
	
	while (true) {
		/**
		 * Set up your blocking queues: Be sure to size these properly based
		 * on expected TPS of your stream
		 */
		BlockingQueue<String> msgQueue = new LinkedBlockingQueue<String>(
				10000);

		/**
		 * Declare the host you want to connect to, the endpoint, and
		 * authentication (basic auth or oauth)
		 */
		StatusesFilterEndpoint endpoint = new StatusesFilterEndpoint();
		
		// Track  anything that is geo-tagged
		endpoint.addQueryParameter("locations", "-180,-90,180,90");

		// These secrets should be read from a config file
		Authentication hosebirdAuth = new OAuth1(consumerKey,
				consumerSecret, token, secret);

		// create a new basic client - by default gzip is enabled
		Client client = new ClientBuilder().hosts(Constants.STREAM_HOST)
				.endpoint(endpoint).authentication(hosebirdAuth)
				.processor(new StringDelimitedProcessor(msgQueue)).build();

		client.connect();
		

		LOG.info("Got connection to Twitter");
		
		// create producer
		ProducerClient producer = new ProducerBuilder()
				.withName("Twitter")
				.withStreamName(streamName)
				.withRegion(regionName)
				.withThreads(10)
				.build();
		
		producer.connect();
		
		LOG.info("Got connection to Kinesis");

		try {
			if (process(msgQueue, producer)) {
				break;
			}

		} catch (Exception e) {
			// if we get here, our client has broken, throw away and
			// recreate
			e.printStackTrace();
		}
		// also, if we make it here, we have had a problem, so start again
		client.stop();
	}
}
 

public static void main(String[] args)  throws InterruptedException {

        BlockingQueue<String> msgQueue = new LinkedBlockingDeque<>();



        Hosts hosebirdHosts = new HttpHosts(Constants.STREAM_HOST);
        StatusesFilterEndpoint hosebirdEndpoint = new StatusesFilterEndpoint();
        List<String> terms = Lists.newArrayList("superman vs batman","#supermanvsbatman");
        hosebirdEndpoint.trackTerms(terms);




        Authentication hosebirdAuth  = new OAuth1("18qydWMuiUohwCtQpp1MOFCFr",
                                                  "YrYhYd09LKZLbhsKT1o4XcEPl6HiAoNykiOxYBq0dAB8t0vRCo",
                                                  "16972669-KSvyDEMc7dussPfW6a9Ru65L4eWGj637ciHLHZLyn",
                                                  "ky53NE6cbBvtNLopto7o9gVyHDejSB2kPsRhHGKEd1MrS");


        ClientBuilder clientBuilder = new ClientBuilder();
        clientBuilder.name("bbejeck-hosebird")
                     .hosts(hosebirdHosts)
                     .authentication(hosebirdAuth)
                     .endpoint(hosebirdEndpoint)
                     .processor(new StringDelimitedProcessor(msgQueue));

        Client hosebirdClient = clientBuilder.build();
        hosebirdClient.connect();

        for (int msgRead = 0; msgRead < 100; msgRead++) {
            String msg = msgQueue.take();
            System.out.println(msg);
        }

        hosebirdClient.stop();

    }