org.apache.storm.windowing.TupleWindow Java Examples

@Override
public void execute(TupleWindow inputWindow) {
    int sum = 0;
    List<Tuple> tuplesInWindow = inputWindow.get();
    LOG.debug("Events in current window: " + tuplesInWindow.size());
    if (tuplesInWindow.size() > 0) {
        /*
         * Since this is a tumbling window calculation,
         * we use all the tuples in the window to compute the avg.
         */
        for (Tuple tuple : tuplesInWindow) {
            sum += (int) tuple.getValue(0);
        }
        collector.emit(new Values(sum / tuplesInWindow.size()));
    }
}
 

@Override
public void execute(TupleWindow inputWindow) {
  int sum = 0;
  List<Tuple> tuplesInWindow = inputWindow.get();
  LOG.fine("Events in current window: " + tuplesInWindow.size());
  if (tuplesInWindow.size() > 0) {
            /*
            * Since this is a tumbling window calculation,
            * we use all the tuples in the window to compute the avg.
            */
    for (Tuple tuple : tuplesInWindow) {
      sum += (int) tuple.getValue(0);
    }
    collector.emit(new Values(sum / tuplesInWindow.size()));
  }
}
 

@Override
public void execute(TupleWindow window) {
    int sum = 0;
    int count = 0;
    // iterate over tuples in the current window
    Iterator<Tuple> it = window.getIter();
    while (it.hasNext()) {
        Tuple tuple = it.next();
        sum += tuple.getInteger(0);
        ++count;
    }
    LOG.debug("Count : {}", count);
    globalAvg = Pair.of(globalAvg.getFirst() + sum, globalAvg.getSecond() + count);
    // update the value in state
    state.put(STATE_KEY, globalAvg);
    // emit the averages downstream
    collector.emit(new Values(new Averages((double) globalAvg.getFirst() / globalAvg.getSecond(), (double) sum / count)));
}
 

/**
 * Process the tuple window and optionally emit new tuples based on the tuples in the input window.
 *
 * @param inputWindow
 */
@Override
public void execute(TupleWindow inputWindow) {
    ++windowId;
    LOG.debug("Window activated, window id {}, number of tuples in window {}", windowId, inputWindow.get().size());

    Map<String, Tuple> eventIdToTupleMap = new HashMap<>();
    try {
        StreamlineEvent event;
        for (Tuple input : inputWindow.get()) {
            if ((event = getStreamlineEventFromTuple(input)) != null) {
                LOG.debug("++++++++ Executing tuple [{}] which contains StreamlineEvent [{}]", input, event);
                eventIdToTupleMap.put(event.getId(), input);
                processAndEmit(event, eventIdToTupleMap);
            }
        }
        // force evaluation of the last group by
        processAndEmit(GROUP_BY_TRIGGER_EVENT, eventIdToTupleMap);

        // current group is processed and result emitted
        eventIdToTupleMap.clear();
    } catch (Exception e) {
        collector.reportError(e);
        LOG.error("", e);
    }
}
 

/**
 * If the {@code FlushSignal} tells the bolt NOT to flush, nothing should be emitted.
 */
@Test
public void testDoNotFlushActiveProfiles() throws Exception {

  ProfileBuilderBolt bolt = createBolt();

  // create a mock where flush() returns the profile measurement above
  MessageDistributor distributor = mock(MessageDistributor.class);
  when(distributor.flush()).thenReturn(Collections.singletonList(measurement));
  bolt.withMessageDistributor(distributor);

  // there is no flush signal
  flushSignal.setFlushNow(false);

  // execute the bolt
  Tuple tuple1 = createTuple("entity1", message1, profile1, 1000L);
  TupleWindow tupleWindow = createWindow(tuple1);
  bolt.execute(tupleWindow);

  // nothing should have been emitted
  getProfileMeasurements(outputCollector, 0);
}
 

/**
 * If the {@code FlushSignal} tells the bolt to flush, it should flush the {@code MessageDistributor}
 * and emit the {@code ProfileMeasurement} values from all active profiles.
 */
@Test
public void testFlushActiveProfiles() throws Exception {

  ProfileBuilderBolt bolt = createBolt();

  // create a mock that returns the profile measurement above
  MessageDistributor distributor = mock(MessageDistributor.class);
  when(distributor.flush()).thenReturn(Collections.singletonList(measurement));
  bolt.withMessageDistributor(distributor);

  // signal the bolt to flush
  flushSignal.setFlushNow(true);

  // execute the bolt
  Tuple tuple1 = createTuple("entity1", message1, profile1, 1000L);
  TupleWindow tupleWindow = createWindow(tuple1);
  bolt.execute(tupleWindow);

  // a profile measurement should be emitted by the bolt
  List<ProfileMeasurement> measurements = getProfileMeasurements(outputCollector, 1);
  assertEquals(1, measurements.size());
  assertEquals(measurement, measurements.get(0));
}
 

/**
 * The bolt should extract a message and timestamp from a tuple and
 * pass that to a {@code MessageDistributor}.
 */
@Test
public void testExtractMessage() throws Exception {

  ProfileBuilderBolt bolt = createBolt();

  // create a mock
  MessageDistributor distributor = mock(MessageDistributor.class);
  bolt.withMessageDistributor(distributor);

  // create a tuple
  final long timestamp1 = 100000000L;
  Tuple tuple1 = createTuple("entity1", message1, profile1, timestamp1);

  // execute the bolt
  TupleWindow tupleWindow = createWindow(tuple1);
  bolt.execute(tupleWindow);

  // the message should have been extracted from the tuple and passed to the MessageDistributor
  verify(distributor).distribute(any(MessageRoute.class), any());
}
 

@Override
public void execute(TupleWindow window) {
  if(LOG.isDebugEnabled()) {
    log(window);
  }

  try {
    // handle each tuple in the window
    for(Tuple tuple : window.get()) {
      handleMessage(tuple);
    }

    // time to flush active profiles?
    if(activeFlushSignal.isTimeToFlush()) {
      flushActive();
    }

  } catch (Throwable e) {
    LOG.error("Unexpected error", e);
    collector.reportError(e);
  }
}
 

private boolean doTest(String rulesJson, int expectedExecuteCount, Function<Integer, Tuple> tupleGen) throws Exception {
    RulesProcessor rulesProcessor = Utils.createObjectFromJson(rulesJson, RulesProcessor.class);
    Window windowConfig = rulesProcessor.getRules().get(0).getWindow();
    final CountDownLatch latch = new CountDownLatch(expectedExecuteCount);
    WindowRulesBolt wb = new WindowRulesBolt(rulesJson, RuleProcessorRuntime.ScriptType.SQL) {
        @Override
        public void execute(TupleWindow inputWindow) {
            super.execute(inputWindow);
            latch.countDown();
        }
    };
    wb.withWindowConfig(windowConfig);
    WindowedBoltExecutor wbe = new WindowedBoltExecutor(wb);
    Map<String, Object> conf = wb.getComponentConfiguration();
    conf.put("topology.message.timeout.secs", 30);
    wbe.prepare(conf, mockContext, mockCollector);
    Thread.sleep(100);
    for (int i = 1; i <= 20; i++) {
        wbe.execute(tupleGen.apply(i));
    }
    // wait for up to 5 secs for the bolt's execute to finish
    return latch.await(5, TimeUnit.SECONDS);
}
 

private static TupleWindow makeTupleWindow(ArrayList<Tuple>... streams) {
    ArrayList<Tuple> combined = null;
    for (int i = 0; i < streams.length; i++) {
        if(i==0) {
            combined = new ArrayList<>(streams[0]);
        } else {
            combined.addAll(streams[i]);
        }
    }
    return new TupleWindowImpl(combined, null, null);
}
 

@Override
public void execute(TupleWindow tupleWindow) {
    List<Tuple> tuples = tupleWindow.get();
    tuples.sort(Comparator.comparing(a -> a.getLongByField("timestamp")));
    //This is safe since the window is calculated basing on Tuple's timestamp, thus it can't really be empty
    Long beginningTimestamp = tuples.get(0).getLongByField("timestamp");
    Long endTimestamp = tuples.get(tuples.size() - 1).getLongByField("timestamp");
    int sumOfOperations = tuples.stream().mapToInt(tuple -> tuple.getIntegerByField("operation")).sum();
    Values values = new Values(sumOfOperations, beginningTimestamp, endTimestamp);
    outputCollector.emit(values);
}
 

@Override
public void execute(TupleWindow inputWindow) {
    for (Tuple tuple : inputWindow.get()) {
        sum += tuple.getIntegerByField("value");
    }
    state.put("sum", sum);
    collector.emit(new Values(sum));
}
 

/**
 * Creates a mock TupleWindow containing multiple tuples.
 * @param tuples The tuples to add to the window.
 */
private TupleWindow createWindow(Tuple... tuples) {

  TupleWindow window = mock(TupleWindow.class);
  when(window.get()).thenReturn(Arrays.asList(tuples));
  return window;
}
 

/**
 * A {@link ProfileMeasurement} is built for each profile/entity pair.  The measurement should be emitted to each
 * destination defined by the profile. By default, a profile uses both Kafka and HBase as destinations.
 */
@Test
public void testEmitters() throws Exception {

  // defines the zk configurations accessible from the bolt
  ProfilerConfigurations configurations = new ProfilerConfigurations();
  configurations.updateGlobalConfig(Collections.emptyMap());

  // create the bolt with 3 destinations
  ProfileBuilderBolt bolt = (ProfileBuilderBolt) new ProfileBuilderBolt()
          .withProfileTimeToLive(30, TimeUnit.MINUTES)
          .withPeriodDuration(10, TimeUnit.MINUTES)
          .withMaxNumberOfRoutes(Long.MAX_VALUE)
          .withZookeeperClient(client)
          .withZookeeperCache(cache)
          .withEmitter(new TestEmitter("destination1"))
          .withEmitter(new TestEmitter("destination2"))
          .withEmitter(new TestEmitter("destination3"))
          .withProfilerConfigurations(configurations)
          .withTumblingWindow(new BaseWindowedBolt.Duration(10, TimeUnit.MINUTES));
  bolt.prepare(new HashMap<>(), topologyContext, outputCollector);

  // signal the bolt to flush
  bolt.withFlushSignal(flushSignal);
  flushSignal.setFlushNow(true);

  // execute the bolt
  Tuple tuple1 = createTuple("entity", message1, profile1, System.currentTimeMillis());
  TupleWindow window = createWindow(tuple1);
  bolt.execute(window);

  // validate measurements emitted to each
  verify(outputCollector, times(1)).emit(eq("destination1"), any());
  verify(outputCollector, times(1)).emit(eq("destination2"), any());
  verify(outputCollector, times(1)).emit(eq("destination3"), any());
}
 

/**
 * Logs information about the {@link TupleWindow}.
 *
 * @param window The tuple window.
 */
private void log(TupleWindow window) {
  // summarize the newly received tuples
  LongSummaryStatistics received = window.get()
          .stream()
          .map(tuple -> getField(TIMESTAMP_TUPLE_FIELD, tuple, Long.class))
          .collect(Collectors.summarizingLong(Long::longValue));

  LOG.debug("Tuple(s) received; count={}, min={}, max={}, range={} ms",
          received.getCount(),
          received.getMin(),
          received.getMax(),
          received.getMax() - received.getMin());

  if (window.getExpired().size() > 0) {
    // summarize the expired tuples
    LongSummaryStatistics expired = window.getExpired()
            .stream()
            .map(tuple -> getField(TIMESTAMP_TUPLE_FIELD, tuple, Long.class))
            .collect(Collectors.summarizingLong(Long::longValue));

    LOG.debug("Tuple(s) expired; count={}, min={}, max={}, range={} ms, lag={} ms",
            expired.getCount(),
            expired.getMin(),
            expired.getMax(),
            expired.getMax() - expired.getMin(),
            received.getMin() - expired.getMin());
  }
}
 

@Override
public void execute(TupleWindow inputWindow) {
  List<Integer> indexesInThisWindow = new ArrayList<>();
  for (Tuple t : inputWindow.get()) {
    indexesInThisWindow.add(t.getInteger(0));
  }
  System.out.println("Tuple received : " + indexesInThisWindow);
}
 

@Test
public void testNestedKeys_StreamLine() throws Exception {
    ArrayList<Tuple> userStream = makeStreamLineEventStream("users", userFields, users);
    ArrayList<Tuple> cityStream = makeStreamLineEventStream("cities", cityFields, cities);
    TupleWindow window = makeTupleWindow(userStream, cityStream);
    WindowedQueryBolt bolt = new WindowedQueryBolt("users", "city")
            .join("cities", "cityName", "users")
            .selectStreamLine("name, users:city as city, cities:country");
    MockTopologyContext context = new MockTopologyContext(new String[]{StreamlineEvent.STREAMLINE_EVENT});
    MockCollector collector = new MockCollector();
    bolt.prepare(null, context, collector);
    bolt.execute(window);
    printResults_StreamLine(collector);
    Assert.assertEquals( cityStream.size(), collector.actualResults.size() );
}
 

@Test
public void testNestedKeys_trivial() throws Exception {
    ArrayList<Tuple> userStream = makeStreamLineEventStream("users", userFields, users);
    TupleWindow window = makeTupleWindow(userStream);
    WindowedQueryBolt bolt = new WindowedQueryBolt("users", SL_PREFIX + "userId")
            .selectStreamLine("name,users:city, users:city as cityagain");
    MockTopologyContext context = new MockTopologyContext(new String[]{StreamlineEvent.STREAMLINE_EVENT});
    MockCollector collector = new MockCollector();
    bolt.prepare(null, context, collector);
    bolt.execute(window);
    printResults_StreamLine(collector);
    Assert.assertEquals( userStream.size(), collector.actualResults.size() );
}
 

@Override
public void execute(TupleWindow inputWindow) {
  List<Integer> indexesInThisWindow = new ArrayList<>();
  for (Tuple t : inputWindow.get()) {
    indexesInThisWindow.add(t.getInteger(0));
  }
  System.out.println("Tuple received : " + indexesInThisWindow);
}
 

@Override
public void execute(TupleWindow inputWindow) {
  List<Integer> indexesInThisWindow = new ArrayList<>();
  for (Tuple t : inputWindow.get()) {
    indexesInThisWindow.add(t.getInteger(0));
  }
  System.out.println("Tuple received : " + indexesInThisWindow);
}
 

private static TupleWindow makeTupleWindow(ArrayList<Tuple> stream) {
    return new TupleWindowImpl(stream, null, null);
}
 

@Override
public void execute(TupleWindow tupleWindow) {
    processorBolt.execute(tupleWindow);
}
 

@Override
public void execute(TupleWindow inputWindow) {
  collector.emit(new Values(inputWindow.get().size()));
}
 

@Override
public void execute(TupleWindow inputWindow) {

}
 

/**
 * Process the tuple window and optionally emit new tuples based on the tuples in the input
 * window.
 */
void execute(TupleWindow inputWindow);
 

/**
 * Process the tuple window and optionally emit new tuples based on the tuples in the input
 * window.
 */
void execute(TupleWindow inputWindow);