Java Code Examples for backtype.storm.topology.TopologyBuilder#createTopology()

@Override
public StormTopology getTopology(Config config) {
  final int numLevels = BenchmarkUtils.getInt(config, TOPOLOGY_LEVEL, DEFAULT_NUM_LEVELS);
  final int msgSize = BenchmarkUtils.getInt(config, RandomMessageSpout.MESSAGE_SIZE,
          RandomMessageSpout.DEFAULT_MESSAGE_SIZE);
  final int spoutNum = BenchmarkUtils.getInt(config, SPOUT_NUM, DEFAULT_SPOUT_NUM);
  final int boltNum = BenchmarkUtils.getInt(config, BOLT_NUM, DEFAULT_BOLT_NUM);

  spout = new RandomMessageSpout(msgSize, BenchmarkUtils.ifAckEnabled(config));

  TopologyBuilder builder = new TopologyBuilder();

  builder.setSpout(SPOUT_ID, spout, spoutNum);
  builder.setBolt(BOLT_ID + 1, new ConstBolt(), boltNum)
      .shuffleGrouping(SPOUT_ID);
  for (int levelNum = 2; levelNum <= numLevels - 1; levelNum++) {
    builder.setBolt(BOLT_ID + levelNum, new ConstBolt(), boltNum)
      .shuffleGrouping(BOLT_ID + (levelNum - 1));
  }
 return builder.createTopology();
}
 

@Override
public StormTopology getTopology(Config config) {

  final int spoutNum = BenchmarkUtils.getInt(config, SPOUT_NUM, DEFAULT_SPOUT_NUM);
  final int matBoltNum = BenchmarkUtils.getInt(config, FM_NUM, DEFAULT_MAT_BOLT_NUM);
  final int cntBoltNum = BenchmarkUtils.getInt(config, CM_NUM, DEFAULT_CNT_BOLT_NUM);
  final String ptnString = (String) Utils.get(config, PATTERN_STRING, DEFAULT_PATTERN_STR);

  spout = new KafkaSpout(KafkaUtils.getSpoutConfig(config, new SchemeAsMultiScheme(new StringScheme())));

  TopologyBuilder builder = new TopologyBuilder();
  builder.setSpout(SPOUT_ID, spout, spoutNum);
  builder.setBolt(FM_ID, new FindMatchingSentence(ptnString), matBoltNum)
          .localOrShuffleGrouping(SPOUT_ID);
  builder.setBolt(CM_ID, new CountMatchingSentence(), cntBoltNum)
          .fieldsGrouping(FM_ID, new Fields(FindMatchingSentence.FIELDS));

  return builder.createTopology();
}
 

@Override
public StormTopology getTopology(Config config) {

  final int spoutNum = BenchmarkUtils.getInt(config, SPOUT_NUM, DEFAULT_SPOUT_NUM);
  final int viewBoltNum = BenchmarkUtils.getInt(config, VIEW_NUM, DEFAULT_VIEW_BOLT_NUM);
  final int cntBoltNum = BenchmarkUtils.getInt(config, COUNT_NUM, DEFAULT_COUNT_BOLT_NUM);

  spout = new KafkaSpout(KafkaUtils.getSpoutConfig(
          config, new SchemeAsMultiScheme(new StringScheme())));

  TopologyBuilder builder = new TopologyBuilder();
  builder.setSpout(SPOUT_ID, spout, spoutNum);
  builder.setBolt(VIEW_ID, new PageViewBolt(Item.URL, Item.ONE), viewBoltNum)
         .localOrShuffleGrouping(SPOUT_ID);
  builder.setBolt(COUNT_ID, new WordCount.Count(), cntBoltNum)
          .fieldsGrouping(VIEW_ID, new Fields(Item.URL.toString()));
  return builder.createTopology();
}
 

@Override
public StormTopology getTopology(Config config) {

  final int spoutNum = BenchmarkUtils.getInt(config, SPOUT_NUM, DEFAULT_SPOUT_NUM);
  final int pvBoltNum = BenchmarkUtils.getInt(config, VIEW_NUM, DEFAULT_PV_BOLT_NUM);
  final int uvBoltNum = BenchmarkUtils.getInt(config, UNIQUER_NUM, DEFAULT_UV_BOLT_NUM);
  final int winLen = BenchmarkUtils.getInt(config, WINDOW_LENGTH, DEFAULT_WINDOW_LENGTH_IN_SEC);
  final int emitFreq = BenchmarkUtils.getInt(config, EMIT_FREQ, DEFAULT_EMIT_FREQ_IN_SEC);
  spout = new KafkaSpout(KafkaUtils.getSpoutConfig(
          config, new SchemeAsMultiScheme(new StringScheme())));

  TopologyBuilder builder = new TopologyBuilder();
  builder.setSpout(SPOUT_ID, spout, spoutNum);
  builder.setBolt(VIEW_ID, new PageViewBolt(Item.URL, Item.USER), pvBoltNum)
          .localOrShuffleGrouping(SPOUT_ID);
  builder.setBolt(UNIQUER_ID, new UniqueVisitorBolt(winLen, emitFreq), uvBoltNum)
          .fieldsGrouping(VIEW_ID, new Fields(Item.URL.toString()));
  return builder.createTopology();
}
 

@Override
public StormTopology execute(Config config, StormEnvironment environment) {
    TopologyBuilder builder = new TopologyBuilder();
    KafkaSpoutProvider provider = new KafkaSpoutProvider();
    IRichSpout spout = provider.getSpout(config);

    HBaseAuditLogParserBolt bolt = new HBaseAuditLogParserBolt();

    int numOfSpoutTasks = config.getInt(SPOUT_TASK_NUM);
    int numOfParserTasks = config.getInt(PARSER_TASK_NUM);
    int numOfJoinTasks = config.getInt(JOIN_TASK_NUM);
    int numOfSinkTasks = config.getInt(SINK_TASK_NUM);

    builder.setSpout("ingest", spout, numOfSpoutTasks);
    BoltDeclarer boltDeclarer = builder.setBolt("parserBolt", bolt, numOfParserTasks);
    boltDeclarer.fieldsGrouping("ingest", new Fields(StringScheme.STRING_SCHEME_KEY));

    HBaseResourceSensitivityDataJoinBolt joinBolt = new HBaseResourceSensitivityDataJoinBolt(config);
    BoltDeclarer joinBoltDeclarer = builder.setBolt("joinBolt", joinBolt, numOfJoinTasks);
    joinBoltDeclarer.fieldsGrouping("parserBolt", new Fields("f1"));

    StormStreamSink sinkBolt = environment.getStreamSink("hbase_audit_log_stream",config);
    BoltDeclarer kafkaBoltDeclarer = builder.setBolt("kafkaSink", sinkBolt, numOfSinkTasks);
    kafkaBoltDeclarer.fieldsGrouping("joinBolt", new Fields("user"));
    return builder.createTopology();
}
 

@Override
public StormTopology execute(Config config, StormEnvironment environment) {
    TopologyBuilder builder = new TopologyBuilder();
    KafkaSpoutProvider provider = new KafkaSpoutProvider();
    IRichSpout spout = provider.getSpout(config);

    int numOfSpoutTasks = config.getInt(SPOUT_TASK_NUM);
    int numOfParserTask = config.getInt(PARSER_TASK_NUM);
    int numOfJoinTasks = config.getInt(JOIN_TASK_NUM);

    builder.setSpout("ingest", spout, numOfSpoutTasks);

    OozieAuditLogParserBolt parserBolt = new OozieAuditLogParserBolt();
    BoltDeclarer parserBoltDeclarer = builder.setBolt("parserBolt", parserBolt, numOfParserTask);
    parserBoltDeclarer.fieldsGrouping("ingest", new Fields(StringScheme.STRING_SCHEME_KEY));

    OozieResourceSensitivityDataJoinBolt joinBolt = new OozieResourceSensitivityDataJoinBolt(config);
    BoltDeclarer boltDeclarer = builder.setBolt("joinBolt", joinBolt, numOfJoinTasks);
    boltDeclarer.fieldsGrouping("parserBolt", new Fields("f1"));

    StormStreamSink sinkBolt = environment.getStreamSink("oozie_audit_log_stream", config);
    int numOfSinkTasks = config.getInt(SINK_TASK_NUM);
    BoltDeclarer kafkaBoltDeclarer = builder.setBolt("kafkaSink", sinkBolt, numOfSinkTasks);
    kafkaBoltDeclarer.fieldsGrouping("joinBolt", new Fields("user"));
    return builder.createTopology();
}
 

@Override
public StormTopology getTopology(Map<String, Object> config) {
    TopologyBuilder builder = new TopologyBuilder();

    // spouts
    FluxShellSpout spout = new FluxShellSpout(
            new String[]{"node", "randomsentence.js"},
            new String[]{"word"});
    builder.setSpout("sentence-spout", spout, 1);

    // bolts
    builder.setBolt("log-bolt", new LogInfoBolt(), 1)
            .shuffleGrouping("sentence-spout");

    return builder.createTopology();
}
 

@Override
public StormTopology getTopology(Config config) {

  final int spoutNum = BenchmarkUtils.getInt(config, SPOUT_NUM, DEFAULT_SPOUT_NUM);
  final int spBoltNum = BenchmarkUtils.getInt(config, SPLIT_NUM, DEFAULT_SP_BOLT_NUM);
  final int rcBoltNum = BenchmarkUtils.getInt(config, COUNTER_NUM, DEFAULT_RC_BOLT_NUM);
  final int windowLength = BenchmarkUtils.getInt(config, WINDOW_LENGTH,
          RollingBolt.DEFAULT_SLIDING_WINDOW_IN_SECONDS);
  final int emitFreq = BenchmarkUtils.getInt(config, EMIT_FREQ,
          RollingBolt.DEFAULT_EMIT_FREQUENCY_IN_SECONDS);

  spout = new FileReadSpout(BenchmarkUtils.ifAckEnabled(config));

  TopologyBuilder builder = new TopologyBuilder();

  builder.setSpout(SPOUT_ID, spout, spoutNum);
  builder.setBolt(SPLIT_ID, new WordCount.SplitSentence(), spBoltNum)
          .localOrShuffleGrouping(SPOUT_ID);
  builder.setBolt(COUNTER_ID, new RollingCountBolt(windowLength, emitFreq), rcBoltNum)
          .fieldsGrouping(SPLIT_ID, new Fields(WordCount.SplitSentence.FIELDS));
  return builder.createTopology();
}
 

@Override
public StormTopology execute(Config config, StormEnvironment environment) {
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("metric_spout", new RandomEventSpout(), config.getInt("spoutNum"));
    builder.setBolt("sink_1",environment.getStreamSink("TEST_STREAM_1",config)).fieldsGrouping("metric_spout",new Fields("metric"));
    builder.setBolt("sink_2",environment.getStreamSink("TEST_STREAM_2",config)).fieldsGrouping("metric_spout",new Fields("metric"));
    return builder.createTopology();
}
 

public StormTopology getTopology(Config config) {
    TopologyBuilder builder = new TopologyBuilder();

    // spouts
    FluxShellSpout spout = new FluxShellSpout(
            new String[]{"node", "randomsentence.js"},
            new String[]{"word"});
    builder.setSpout("sentence-spout", spout, 1);

    // bolts
    builder.setBolt("log-bolt", new LogInfoBolt(), 1)
            .shuffleGrouping("sentence-spout");

    return builder.createTopology();
}
 

public static void main(String[] args) {
  TopologyBuilder builder = new TopologyBuilder();

  builder.setSpout("commit-feed-listener", new CommitFeedListener());

  builder
      .setBolt("email-extractor", new EmailExtractor())
      .shuffleGrouping("commit-feed-listener");

  builder
      .setBolt("email-counter", new EmailCounter())
      .fieldsGrouping("email-extractor", new Fields("email"));

  Config config = new Config();
  config.setDebug(true);

  StormTopology topology = builder.createTopology();

  LocalCluster cluster = new LocalCluster();
  cluster.submitTopology("github-commit-count-topology",
      config,
      topology);

  Utils.sleep(TEN_MINUTES);
  cluster.killTopology("github-commit-count-topology");
  cluster.shutdown();
}
 

@Override
public StormTopology execute(Config config, StormEnvironment environment) {
    TopologyBuilder builder = new TopologyBuilder();
    HiveJobRunningSourcedStormSpoutProvider provider = new HiveJobRunningSourcedStormSpoutProvider();
    IRichSpout spout = provider.getSpout(config, config.getInt(SPOUT_TASK_NUM));


    int numOfSpoutTasks = config.getInt(SPOUT_TASK_NUM);
    int numOfFilterTasks = config.getInt(FILTER_TASK_NUM);
    int numOfParserTasks = config.getInt(PARSER_TASK_NUM);
    int numOfJoinTasks = config.getInt(JOIN_TASK_NUM);
    int numOfSinkTasks = config.getInt(SINK_TASK_NUM);

    builder.setSpout("ingest", spout, numOfSpoutTasks);
    JobFilterBolt bolt = new JobFilterBolt();
    BoltDeclarer boltDeclarer = builder.setBolt("filterBolt", bolt, numOfFilterTasks);
    boltDeclarer.fieldsGrouping("ingest", new Fields("jobId"));

    HiveQueryParserBolt parserBolt = new HiveQueryParserBolt();
    BoltDeclarer parserBoltDeclarer = builder.setBolt("parserBolt", parserBolt, numOfParserTasks);
    parserBoltDeclarer.fieldsGrouping("filterBolt", new Fields("user"));

    HiveSensitivityDataEnrichBolt joinBolt = new HiveSensitivityDataEnrichBolt(config);
    BoltDeclarer joinBoltDeclarer = builder.setBolt("joinBolt", joinBolt, numOfJoinTasks);
    joinBoltDeclarer.fieldsGrouping("parserBolt", new Fields("user"));

    StormStreamSink sinkBolt = environment.getStreamSink("hive_query_stream", config);
    BoltDeclarer kafkaBoltDeclarer = builder.setBolt("kafkaSink", sinkBolt, numOfSinkTasks);
    kafkaBoltDeclarer.fieldsGrouping("joinBolt", new Fields("user"));
    return builder.createTopology();
}
 

public static void main(String[] args) throws AlreadyAliveException, InvalidTopologyException {
    Map config = new Config();
    config.put(ConfigExtension.TOPOLOGY_MASTER_USER_DEFINED_STREAM_CLASS, "com.alipay.dw.jstorm.example.tm.TMUdfHandler");
    config.put(Config.TOPOLOGY_WORKERS, 2);

    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("TMUdfSpout", new TMUdfSpout(), 2);
    builder.setBolt("TMUdfBolt", new TMUdfBolt(), 4);
    StormTopology topology = builder.createTopology();

    StormSubmitter.submitTopology("TMUdfTopology", config, topology);
}
 

public StormTopology execute(Config config, StormEnvironment environment) {
    HadoopQueueRunningAppConfig appConfig = new HadoopQueueRunningAppConfig(config);

    String spoutName = "runningQueueSpout";
    String persistBoltName = "persistBolt";

    IRichSpout spout = new HadoopQueueRunningSpout(appConfig);

    //String acceptedAppStreamId = persistBoltName + "-to-" + DataSource.RUNNING_APPS.toString();
    //String schedulerStreamId = persistBoltName + "-to-" + DataSource.SCHEDULER.toString();
    //streamMaps.put(DataSource.RUNNING_APPS, acceptedAppStreamId);
    //streamMaps.put(DataSource.SCHEDULER, schedulerStreamId);

    int numOfPersistTasks = appConfig.topology.numPersistTasks;
    int numOfSinkTasks = appConfig.topology.numSinkTasks;
    int numOfSpoutTasks = 1;

    HadoopQueueMetricPersistBolt bolt = new HadoopQueueMetricPersistBolt(appConfig);
    TopologyBuilder builder = new TopologyBuilder();

    builder.setSpout(spoutName, spout, numOfSpoutTasks).setNumTasks(numOfSpoutTasks);
    builder.setBolt(persistBoltName, bolt, numOfPersistTasks).setNumTasks(numOfPersistTasks).shuffleGrouping(spoutName);

    StormStreamSink queueSinkBolt = environment.getStreamSink("HADOOP_QUEUE_STREAM", config);
    builder.setBolt("queueKafkaSink", queueSinkBolt, numOfSinkTasks)
            .setNumTasks(numOfSinkTasks).shuffleGrouping(persistBoltName);

    //StormStreamSink appSinkBolt = environment.getStreamSink("ACCEPTED_APP_STREAM", config);
    //builder.setBolt("appKafkaSink", appSinkBolt, numOfSinkTasks)
    //        .setNumTasks(numOfSinkTasks).shuffleGrouping(persistBoltName, acceptedAppStreamId);

    return builder.createTopology();
}
 

@Override
public StormTopology execute(Config config, StormEnvironment environment) {
    //1. trigger prepare conf
    SparkRunningJobAppConfig sparkRunningJobAppConfig = SparkRunningJobAppConfig.newInstance(config);

    //2. prepare topology
    TopologyBuilder topologyBuilder = new TopologyBuilder();
    final String spoutName = SparkRunningJobAppConfig.JOB_FETCH_SPOUT_NAME;
    final String boltName = SparkRunningJobAppConfig.JOB_PARSE_BOLT_NAME;
    int parallelism = sparkRunningJobAppConfig.getJobExtractorConfig().jobFetchSpoutParallism;
    int tasks = sparkRunningJobAppConfig.getJobExtractorConfig().jobFetchSpoutTasksNum;
    if (parallelism > tasks) {
        parallelism = tasks;
    }
    topologyBuilder.setSpout(
            spoutName,
            new SparkRunningJobFetchSpout(
                    sparkRunningJobAppConfig.getJobExtractorConfig(),
                    sparkRunningJobAppConfig.getEndpointConfig(),
                    sparkRunningJobAppConfig.getZkStateConfig()),
            parallelism
    ).setNumTasks(tasks);

    parallelism = sparkRunningJobAppConfig.getJobExtractorConfig().jobParseBoltParallism;
    tasks = sparkRunningJobAppConfig.getJobExtractorConfig().jobParseBoltTasksNum;
    if (parallelism > tasks) {
        parallelism = tasks;
    }
    topologyBuilder.setBolt(boltName,
            new SparkRunningJobParseBolt(
                    sparkRunningJobAppConfig.getZkStateConfig(),
                    sparkRunningJobAppConfig.getEagleServiceConfig(),
                    sparkRunningJobAppConfig.getEndpointConfig(),
                    sparkRunningJobAppConfig.getJobExtractorConfig()),
            parallelism).setNumTasks(tasks).fieldsGrouping(spoutName, new Fields("appId"));

    return topologyBuilder.createTopology();
}
 

private StormTopology createTopology()
        throws AlreadyAliveException, InvalidTopologyException {
    TopologyBuilder builder = new TopologyBuilder();

    builder.setSpout("word", new TestWordSpout(), 10);
    builder.setBolt("exclaim1", new ExclamationBolt(), 3).shuffleGrouping("word");
    builder.setBolt("exclaim2", new ExclamationBolt(), 2).shuffleGrouping("exclaim1");
    
    return builder.createTopology();
}
 

@Override
public StormTopology execute(Config config, StormEnvironment environment) {
    List<String> metricNames = new ArrayList<>();
    String[] metricNamesArr = config.getString("aggregate.counters.metrics").split(",");
    for (int i = 0; i < metricNamesArr.length; i++) {
        metricNames.add(metricNamesArr[i].trim().toLowerCase());
    }
    List<String> groupByColumns = new ArrayList<>();

    String[] groupByColumnsArr = config.getString("aggregate.counters.groupBys").split(";");
    for (int i = 0; i < groupByColumnsArr.length; i++) {
        groupByColumns.add(groupByColumnsArr[i].trim());
    }

    Map<String, List<List<String>>> metrics = new HashMap<>();
    for (String metric : metricNames) {
        metrics.put(metric, new ArrayList<>());
        for (String cols : groupByColumns) {
            metrics.get(metric).add(Arrays.asList(cols.replaceAll(" ", "").split(",")));
        }
    }

    TopologyBuilder topologyBuilder = new TopologyBuilder();
    String spoutName = "mrHistoryAggregationSpout";
    String boltName = "mrHistoryAggregationBolt";
    AggregationConfig aggregationConfig = AggregationConfig.newInstance(config);
    int tasks = aggregationConfig.getConfig().getInt("stormConfig." + spoutName + "Tasks");
    topologyBuilder.setSpout(
        spoutName,
        new AggregationSpout(aggregationConfig, new MRMetricsAggregateContainer(metrics, aggregationConfig)),
        tasks
    ).setNumTasks(tasks);

    tasks = aggregationConfig.getConfig().getInt("stormConfig." + boltName + "Tasks");
    topologyBuilder.setBolt(boltName,
        new AggregationBolt(aggregationConfig.getStormConfig(), new MRMetricsAggregateContainer(metrics, aggregationConfig)),
        tasks).setNumTasks(tasks).shuffleGrouping(spoutName);

    return topologyBuilder.createTopology();
}
 

@Ignore  // this test need zookeeper
@Test
public void testConfigNotify() throws Exception {
    final String topoId = "myTopology";
    int numGroupbyBolts = 2;
    int numTotalGroupbyBolts = 3;
    System.setProperty("correlation.serviceHost", "sandbox.hortonworks.com");
    System.setProperty("correlation.servicePort", "58080");
    System.setProperty("withMetadataChangeNotifyService.zkConfig.zkQuorum", "sandbox.hortonworks.com:2181");
    System.setProperty("correlation.numGroupbyBolts", String.valueOf(numGroupbyBolts));
    System.setProperty("correlation.topologyName", topoId);
    System.setProperty("correlation.mode", "local");
    System.setProperty("correlation.zkHosts", "sandbox.hortonworks.com:2181");
    final String topicName1 = "testTopic3";
    final String topicName2 = "testTopic4";
    // ensure topic ready
    LogManager.getLogger(CorrelationSpout.class).setLevel(Level.DEBUG);
    Config config = ConfigFactory.load();

    CreateTopicUtils.ensureTopicReady(System.getProperty("withMetadataChangeNotifyService.zkConfig.zkQuorum"), topicName1);
    CreateTopicUtils.ensureTopicReady(System.getProperty("withMetadataChangeNotifyService.zkConfig.zkQuorum"), topicName2);

    TopologyBuilder topoBuilder = new TopologyBuilder();

    int numBolts = config.getInt("correlation.numGroupbyBolts");
    String spoutId = "correlation-spout";
    CorrelationSpout spout = new CorrelationSpout(config, topoId,
        new MockMetadataChangeNotifyService(topoId, spoutId), numBolts);
    SpoutDeclarer declarer = topoBuilder.setSpout(spoutId, spout);
    declarer.setNumTasks(2);
    for (int i = 0; i < numBolts; i++) {
        TestBolt bolt = new TestBolt();
        BoltDeclarer boltDecl = topoBuilder.setBolt("engineBolt" + i, bolt);
        boltDecl.fieldsGrouping(spoutId,
            StreamIdConversion.generateStreamIdBetween(AlertConstants.DEFAULT_SPOUT_NAME, AlertConstants.DEFAULT_ROUTERBOLT_NAME + i), new Fields());
    }

    String topoName = config.getString("correlation.topologyName");
    LOG.info("start topology in local mode");
    LocalCluster cluster = new LocalCluster();
    StormTopology topology = topoBuilder.createTopology();
    cluster.submitTopology(topoName, new HashMap(), topology);


    Utils.sleep(Long.MAX_VALUE);
}
 

@Override
public StormTopology execute(Config config, StormEnvironment environment) {
    //1. trigger prepare conf
    MRHistoryJobConfig appConfig = MRHistoryJobConfig.newInstance(config);
    com.typesafe.config.Config jhfAppConf = appConfig.getConfig();

    //2. prepare JobHistoryContentFilter
    final JobHistoryContentFilterBuilder builder = JobHistoryContentFilterBuilder.newBuilder().acceptJobFile().acceptJobConfFile();
    String[] confKeyPatternsSplit = jhfAppConf.getString("MRConfigureKeys.jobConfigKey").split(",");
    List<String> confKeyPatterns = new ArrayList<>(confKeyPatternsSplit.length);
    for (String confKeyPattern : confKeyPatternsSplit) {
        confKeyPatterns.add(confKeyPattern.trim());
    }
    confKeyPatterns.add(Constants.JobConfiguration.CASCADING_JOB);
    confKeyPatterns.add(Constants.JobConfiguration.HIVE_JOB);
    confKeyPatterns.add(Constants.JobConfiguration.PIG_JOB);
    confKeyPatterns.add(Constants.JobConfiguration.SCOOBI_JOB);

    String jobNameKey = jhfAppConf.getString("MRConfigureKeys.jobNameKey");
    builder.setJobNameKey(jobNameKey);

    for (String key : confKeyPatterns) {
        builder.includeJobKeyPatterns(Pattern.compile(key));
    }
    JobHistoryContentFilter filter = builder.build();
    //3. prepare topology
    TopologyBuilder topologyBuilder = new TopologyBuilder();
    String spoutName = "mrHistoryJobSpout";
    int tasks = jhfAppConf.getInt("stormConfig.mrHistoryJobSpoutTasks");
    JobHistorySpout jobHistorySpout = new JobHistorySpout(filter, appConfig);
    topologyBuilder.setSpout(
            spoutName,
            jobHistorySpout,
            tasks
    ).setNumTasks(tasks);

    StormStreamSink jobSinkBolt = environment.getStreamSink("MAP_REDUCE_JOB_STREAM", config);
    String jobSinkBoltName = "JobKafkaSink";
    BoltDeclarer jobKafkaBoltDeclarer = topologyBuilder.setBolt(jobSinkBoltName, jobSinkBolt, jhfAppConf.getInt("stormConfig.jobKafkaSinkTasks"))
            .setNumTasks(jhfAppConf.getInt("stormConfig.jobKafkaSinkTasks"));
    String spoutToJobSinkName = spoutName + "_to_" + jobSinkBoltName;
    jobKafkaBoltDeclarer.shuffleGrouping(spoutName, spoutToJobSinkName);

    StormStreamSink taskAttemptSinkBolt = environment.getStreamSink("MAP_REDUCE_TASK_ATTEMPT_STREAM", config);
    String taskAttemptSinkBoltName = "TaskAttemptKafkaSink";
    BoltDeclarer taskAttemptKafkaBoltDeclarer = topologyBuilder.setBolt(taskAttemptSinkBoltName, taskAttemptSinkBolt, jhfAppConf.getInt("stormConfig.taskAttemptKafkaSinkTasks"))
            .setNumTasks(jhfAppConf.getInt("stormConfig.taskAttemptKafkaSinkTasks"));
    String spoutToTaskAttemptSinkName = spoutName + "_to_" + taskAttemptSinkBoltName;
    taskAttemptKafkaBoltDeclarer.shuffleGrouping(spoutName, spoutToTaskAttemptSinkName);

    List<StreamPublisher> streamPublishers = new ArrayList<>();
    //streamPublishers.add(new JobStreamPublisher(spoutToJobSinkName));
    streamPublishers.add(new TaskAttemptStreamPublisher(spoutToTaskAttemptSinkName));
    streamPublishers.add(new JobRpcAnalysisStreamPublisher(spoutToJobSinkName));
    jobHistorySpout.setStreamPublishers(streamPublishers);

    return topologyBuilder.createTopology();
}
 

@Override
public StormTopology execute(Config config, StormEnvironment environment) {
    //1. trigger prepare conf
    MRRunningJobConfig mrRunningJobConfig = MRRunningJobConfig.newInstance(config);

    String[] confKeyPatternsSplit = mrRunningJobConfig.getConfig().getString("MRConfigureKeys.jobConfigKey").split(",");
    List<String> confKeyKeys = new ArrayList<>(confKeyPatternsSplit.length);
    for (String confKeyPattern : confKeyPatternsSplit) {
        confKeyKeys.add(confKeyPattern.trim());
    }
    confKeyKeys.add(Constants.JobConfiguration.CASCADING_JOB);
    confKeyKeys.add(Constants.JobConfiguration.HIVE_JOB);
    confKeyKeys.add(Constants.JobConfiguration.PIG_JOB);
    confKeyKeys.add(Constants.JobConfiguration.SCOOBI_JOB);
    confKeyKeys.add(0, mrRunningJobConfig.getConfig().getString("MRConfigureKeys.jobNameKey"));

    //2. prepare topology
    TopologyBuilder topologyBuilder = new TopologyBuilder();
    String spoutName = "mrRunningJobFetchSpout";
    String boltName = "mrRunningJobParseBolt";
    int tasks = mrRunningJobConfig.getConfig().getInt("stormConfig." + spoutName + "Tasks");

    topologyBuilder.setSpout(
        spoutName,
        new MRRunningJobFetchSpout(mrRunningJobConfig.getEndpointConfig(),
                mrRunningJobConfig.getZkStateConfig()),
        tasks
    ).setNumTasks(tasks);

    tasks = mrRunningJobConfig.getConfig().getInt("stormConfig." + boltName + "Tasks");

    topologyBuilder.setBolt(boltName,
        new MRRunningJobParseBolt(
            mrRunningJobConfig.getEagleServiceConfig(),
            mrRunningJobConfig.getEndpointConfig(),
            mrRunningJobConfig.getZkStateConfig(),
            confKeyKeys,
            config),
            tasks).setNumTasks(tasks).fieldsGrouping(spoutName, APP_TO_JOB_STREAM, new Fields("appId"));

    // parse running/accepted app metrics
    topologyBuilder.setBolt("mrRunningJobMetricBolt", new MRRunningAppMetricBolt(mrRunningJobConfig), 1)
            .setNumTasks(1).shuffleGrouping(spoutName, APP_TO_METRIC_STREAM);
    topologyBuilder.setBolt("acceptedAppSink", environment.getStreamSink("ACCEPTED_APP_STREAM", config), 1)
            .setNumTasks(1).shuffleGrouping("mrRunningJobMetricBolt");

    return topologyBuilder.createTopology();
}