org.apache.storm.topology.base.BaseWindowedBolt Java Examples

public Twister2Bolt(String id, Object bolt, MadeASourceListener madeASourceListener) {
  this.id = id;
  this.boltDeclarer = new Twister2BoltDeclarer(madeASourceListener);
  this.outFieldsForEdge = new EdgeFieldMap(Utils.getDefaultStream(id));
  this.keyedOutEdges = new EdgeFieldMap(Utils.getDefaultStream(id));
  if (bolt instanceof IRichBolt) {
    this.stormBolt = (IRichBolt) bolt;
    this.stormBolt.declareOutputFields(this.outFieldsForEdge);
  } else if (bolt instanceof BaseWindowedBolt) {
    this.stormWindowedBolt = (BaseWindowedBolt) bolt;
    this.stormWindowedBolt.declareOutputFields(this.outFieldsForEdge);
    this.stormWindowedBoltExecutor = new WindowedBoltExecutor(this.stormWindowedBolt);
  } else {
    this.stormBasicBolt = (IBasicBolt) bolt;
    this.stormBasicBolt.declareOutputFields(this.outFieldsForEdge);
  }
}
 

public static void main(String[] args) throws Exception {
  TopologyBuilder builder = new TopologyBuilder();
  builder.setSpout("integer", new RandomIntegerSpout(), 1);
  builder.setBolt("slidingsum", new SlidingWindowSumBolt()
      .withWindow(BaseWindowedBolt.Count.of(30), BaseWindowedBolt.Count.of(10)), 1)
      .shuffleGrouping("integer");
  builder.setBolt("tumblingavg", new TumblingWindowAvgBolt()
      .withTumblingWindow(BaseWindowedBolt.Count.of(3)), 1)
      .shuffleGrouping("slidingsum");
  builder.setBolt("printer", new PrinterBolt(), 1)
      .shuffleGrouping("tumblingavg");
  Config conf = new Config();
  conf.setDebug(true);
  String topoName = "test";

  if (args != null && args.length > 0) {
    topoName = args[0];
  }
  StormSubmitter.submitTopology(topoName, conf, builder.createTopology());
}
 

/**
 * Create a ProfileBuilderBolt to test.
 * @return A {@link ProfileBuilderBolt} to test.
 */
private ProfileBuilderBolt createBolt() throws IOException {

  // defines the zk configurations accessible from the bolt
  ProfilerConfigurations configurations = new ProfilerConfigurations();
  configurations.updateGlobalConfig(Collections.emptyMap());

  emitter = new HBaseEmitter();
  ProfileBuilderBolt bolt = (ProfileBuilderBolt) new ProfileBuilderBolt()
          .withProfileTimeToLive(30, TimeUnit.MINUTES)
          .withMaxNumberOfRoutes(Long.MAX_VALUE)
          .withZookeeperClient(client)
          .withZookeeperCache(cache)
          .withEmitter(emitter)
          .withProfilerConfigurations(configurations)
          .withPeriodDuration(1, TimeUnit.MINUTES)
          .withTumblingWindow(new BaseWindowedBolt.Duration(30, TimeUnit.SECONDS));
  bolt.prepare(new HashMap<>(), topologyContext, outputCollector);

  // set the flush signal AFTER calling 'prepare'
  bolt.withFlushSignal(flushSignal);

  return bolt;
}
 

public static void main(String[] args) throws Exception {
    TopologyBuilder builder = new TopologyBuilder();
    BaseWindowedBolt bolt = new SlidingWindowSumBolt()
        .withWindow(new Duration(5, TimeUnit.SECONDS), new Duration(3, TimeUnit.SECONDS))
        .withTimestampField("ts")
        .withLag(new Duration(5, TimeUnit.SECONDS));
    builder.setSpout("integer", new RandomIntegerSpout(), 1);
    builder.setBolt("slidingsum", bolt, 1).shuffleGrouping("integer");
    builder.setBolt("printer", new PrinterBolt(), 1).shuffleGrouping("slidingsum");
    Config conf = new Config();
    conf.setDebug(true);
    String topoName = "test";

    if (args != null && args.length > 0) {
        topoName = args[0];
    }

    conf.setNumWorkers(1);
    StormSubmitter.submitTopologyWithProgressBar(topoName, conf, builder.createTopology());
}
 

@SuppressWarnings("unchecked")
public static void main(String[] args) throws Exception {
    StreamBuilder builder = new StreamBuilder();
    /**
     * Computes average of the stream of numbers emitted by the spout. Internally the per-partition
     * sum and counts are accumulated and emitted to a downstream task where the partially accumulated
     * results are merged and the final result is emitted.
     */
    builder.newStream(new RandomIntegerSpout(), new ValueMapper<Integer>(0), 2)
           .window(TumblingWindows.of(BaseWindowedBolt.Duration.seconds(5)))
           .filter(x -> x > 0 && x < 500)
           .aggregate(new Avg())
           .print();

    Config config = new Config();
    String topoName = "AGG_EXAMPLE";
    if (args.length > 0) {
        topoName = args[0];
    }
    config.setNumWorkers(1);
    StormSubmitter.submitTopologyWithProgressBar(topoName, config, builder.build());
}
 

public static void runTopology() {
    String filePath = "./src/main/resources/operations.txt";
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("randomNumberSpout", new RandomNumberSpout());
    builder.setBolt("filteringBolt", new FilteringBolt()).shuffleGrouping("randomNumberSpout");
    builder.setBolt("aggregatingBolt", new AggregatingBolt()
      .withTimestampField("timestamp")
      .withLag(BaseWindowedBolt.Duration.seconds(1))
      .withWindow(BaseWindowedBolt.Duration.seconds(5))).shuffleGrouping("filteringBolt");
    builder.setBolt("fileBolt", new FileWritingBolt(filePath)).shuffleGrouping("aggregatingBolt");

    Config config = new Config();
    config.setDebug(false);
    LocalCluster cluster = new LocalCluster();
    cluster.submitTopology("Test", config, builder.createTopology());
}
 

@Override
public StormTopology buildTopology() {
  TopologyBuilder builder = new TopologyBuilder();

  builder.setSpout("source", new TestWordSpout(), 1);
  builder.setBolt("windower", new IntervalWindowBolt()
      .withTumblingWindow(
          new BaseWindowedBolt.Duration(2, TimeUnit.SECONDS)
      ), 1).shuffleGrouping("source");
  return builder.createTopology();
}
 

@Override
public StormTopology buildTopology() {
  TopologyBuilder builder = new TopologyBuilder();

  builder.setSpout("source", new TestWordSpout(), 1);
  builder.setBolt("windower", new TumblingWindowBolt()
      .withTumblingWindow(
          new BaseWindowedBolt.Count(10)
      ), 1)
      .shuffleGrouping("source");
  return builder.createTopology();
}
 

@Override
public StormTopology buildTopology() {
  TopologyBuilder builder = new TopologyBuilder();

  builder.setSpout("source", new TestWordSpout(), 1);
  builder.setBolt("windower", new SlidingWindowBolt()
      .withWindow(
          new BaseWindowedBolt.Count(30),
          new BaseWindowedBolt.Count(10)
      ), 1)
      .shuffleGrouping("source");
  return builder.createTopology();
}
 

public static void main(String[] args) throws Exception {

        SquaresSpout squares = new SquaresSpout();
        CubesSpout cubes = new CubesSpout();


        TopologyBuilder builder = new TopologyBuilder();
        builder.setSpout("squares", squares, 1);
        builder.setSpout("cubes", cubes, 1);

        BaseWindowedBolt joiner = new WindowedQueryBolt("cubes", "number")
                .leftJoin("squares", "number", "cubes")
                .select("number,square,cube")
                .withTumblingWindow(BaseWindowedBolt.Count.of(1000))
                ;

        builder.setBolt("joiner", joiner, 2)
                .fieldsGrouping("squares", new Fields("number") )
                .fieldsGrouping("cubes", new Fields("number") )
        ;


        builder.setBolt("fileWrite", new FileWriteBolt(), 1).shuffleGrouping("joiner");

        // - submit topo

//        LocalCluster cluster = runOnLocalCluster(TOPO_NAME, builder.createTopology() );
//        Thread.sleep(10 * 60*1000);
//        System.err.println("Shutting down");
//        cluster.killTopology(TOPO_NAME);
//        cluster.shutdown();

    }
 

@Override
public BaseWindowedBolt withTumblingWindow(BaseWindowedBolt.Duration duration) {

  // need to capture the window duration to validate it along with other profiler settings
  this.windowDurationMillis = duration.value;
  return super.withTumblingWindow(duration);
}
 

/**
 * A {@link ProfileMeasurement} is built for each profile/entity pair.  The measurement should be emitted to each
 * destination defined by the profile. By default, a profile uses both Kafka and HBase as destinations.
 */
@Test
public void testEmitters() throws Exception {

  // defines the zk configurations accessible from the bolt
  ProfilerConfigurations configurations = new ProfilerConfigurations();
  configurations.updateGlobalConfig(Collections.emptyMap());

  // create the bolt with 3 destinations
  ProfileBuilderBolt bolt = (ProfileBuilderBolt) new ProfileBuilderBolt()
          .withProfileTimeToLive(30, TimeUnit.MINUTES)
          .withPeriodDuration(10, TimeUnit.MINUTES)
          .withMaxNumberOfRoutes(Long.MAX_VALUE)
          .withZookeeperClient(client)
          .withZookeeperCache(cache)
          .withEmitter(new TestEmitter("destination1"))
          .withEmitter(new TestEmitter("destination2"))
          .withEmitter(new TestEmitter("destination3"))
          .withProfilerConfigurations(configurations)
          .withTumblingWindow(new BaseWindowedBolt.Duration(10, TimeUnit.MINUTES));
  bolt.prepare(new HashMap<>(), topologyContext, outputCollector);

  // signal the bolt to flush
  bolt.withFlushSignal(flushSignal);
  flushSignal.setFlushNow(true);

  // execute the bolt
  Tuple tuple1 = createTuple("entity", message1, profile1, System.currentTimeMillis());
  TupleWindow window = createWindow(tuple1);
  bolt.execute(window);

  // validate measurements emitted to each
  verify(outputCollector, times(1)).emit(eq("destination1"), any());
  verify(outputCollector, times(1)).emit(eq("destination2"), any());
  verify(outputCollector, times(1)).emit(eq("destination3"), any());
}
 

public static void main(String[] args) throws Exception {
    StreamBuilder builder = new StreamBuilder();
    // a stream of words
    builder.newStream(new TestWordSpout(), new ValueMapper<String>(0), 2)
           .window(TumblingWindows.of(BaseWindowedBolt.Duration.seconds(2)))
           /*
            * create a stream of (word, 1) pairs
            */
           .mapToPair(w -> Pair.of(w, 1))
           /*
            * compute the word counts in the last two second window
            */
           .countByKey()
           /*
            * update the word counts in the state.
            * Here the first argument 0L is the initial value for the state
            * and the second argument is a function that adds the count to the current value in the state.
            */
           .updateStateByKey(0L, (state, count) -> state + count)
           /*
            * convert the state back to a stream and print the results
            */
           .toPairStream()
           .print();

    Config config = new Config();
    // use redis based state store for persistence
    config.put(Config.TOPOLOGY_STATE_PROVIDER, "org.apache.storm.redis.state.RedisKeyValueStateProvider");
    String topoName = "test";
    if (args.length > 0) {
        topoName = args[0];
    }
    config.setNumWorkers(1);
    StormSubmitter.submitTopologyWithProgressBar(topoName, config, builder.build());
}
 

public static void main(String[] args) throws Exception {
    if (!NimbusClient.isLocalOverride()) {
        throw new IllegalStateException("This example only works in local mode.  "
                                        + "Run with storm local not storm jar");
    }
    FeederSpout genderSpout = new FeederSpout(new Fields("id", "gender"));
    FeederSpout ageSpout = new FeederSpout(new Fields("id", "age"));

    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("genderSpout", genderSpout);
    builder.setSpout("ageSpout", ageSpout);

    // inner join of 'age' and 'gender' records on 'id' field
    JoinBolt joiner = new JoinBolt("genderSpout", "id")
        .join("ageSpout", "id", "genderSpout")
        .select("genderSpout:id,ageSpout:id,gender,age")
        .withTumblingWindow(new BaseWindowedBolt.Duration(10, TimeUnit.SECONDS));

    builder.setBolt("joiner", joiner)
           .fieldsGrouping("genderSpout", new Fields("id"))
           .fieldsGrouping("ageSpout", new Fields("id"));

    builder.setBolt("printer", new PrinterBolt()).shuffleGrouping("joiner");

    Config conf = new Config();
    StormSubmitter.submitTopologyWithProgressBar("join-example", conf, builder.createTopology());

    generateGenderData(genderSpout);

    generateAgeData(ageSpout);
}
 

public static void main( String[] args ) throws Exception {
    String propertiesFile = DEFAULT_PROPERTIES_FILE;
    if (args != null && args.length == 1 && args[0] != null) {
        propertiesFile = args[0];
    }

    LogLevelCountProperties props = new LogLevelCountProperties(propertiesFile);

    int windowMillis = props.getStormWindowMillis();
    double rateThreshold = props.getStormRateThreshold();

    // Build the spout for pulling data from NiFi and pull out the log level into a tuple field
    NiFiSpout niFiSpout = new NiFiSpout(getSourceConfig(props), Collections.singletonList(props.getLogLevelAttribute()));

    // Build the bolt for counting log levels over a tumbling window
    BaseWindowedBolt logLevelWindowBolt = new LogLevelWindowBolt(props.getLogLevelAttribute())
            .withTumblingWindow(new BaseWindowedBolt.Duration(windowMillis, TimeUnit.MILLISECONDS));

    // Build the bolt for pushing results back to NiFi
    NiFiDataPacketBuilder dictionaryBuilder = new DictionaryBuilder(windowMillis, rateThreshold);
    NiFiBolt niFiBolt = new NiFiBolt(getSinkConfig(props), dictionaryBuilder, 10).withBatchSize(1);

    // Build the topology of NiFiSpout -> LogLevelWindowBolt -> NiFiBolt
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("nifiInput", niFiSpout);
    builder.setBolt("logLevels", logLevelWindowBolt).shuffleGrouping("nifiInput");
    builder.setBolt("nifiOutput", niFiBolt).shuffleGrouping("logLevels");

    // Submit the topology
    Config conf = new Config();
    conf.setDebug(true);

    // Need to set the message timeout to twice the window size in seconds
    conf.setMessageTimeoutSecs((props.getStormWindowMillis()/1000) * 2);

    if (args != null && args.length > 0) {
        conf.setNumWorkers(3);
        StormSubmitter.submitTopologyWithProgressBar(args[0], conf, builder.createTopology());
    }
    else {
        LocalCluster cluster = new LocalCluster();
        cluster.submitTopology("log-levels", conf, builder.createTopology());
        Utils.sleep(130000);
        cluster.killTopology("log-levels");
        cluster.shutdown();
    }
}
 

public TestRunWindowProcessorBolt(BaseWindowedBolt processorBolt, String eventLogFilePath) {
    this.processorBolt = processorBolt;
    this.eventLogFilePath = eventLogFilePath;
}