Java Code Examples for backtype.storm.topology.TopologyBuilder#setBolt()

@Override
public StormTopology execute(Config config, StormEnvironment environment) {
    TopologyBuilder builder = new TopologyBuilder();
    KafkaSpoutProvider provider = new KafkaSpoutProvider();
    IRichSpout spout = provider.getSpout(config);

    int numOfSpoutTasks = config.getInt(SPOUT_TASK_NUM);
    int numOfParserTask = config.getInt(PARSER_TASK_NUM);
    int numOfJoinTasks = config.getInt(JOIN_TASK_NUM);

    builder.setSpout("ingest", spout, numOfSpoutTasks);

    OozieAuditLogParserBolt parserBolt = new OozieAuditLogParserBolt();
    BoltDeclarer parserBoltDeclarer = builder.setBolt("parserBolt", parserBolt, numOfParserTask);
    parserBoltDeclarer.fieldsGrouping("ingest", new Fields(StringScheme.STRING_SCHEME_KEY));

    OozieResourceSensitivityDataJoinBolt joinBolt = new OozieResourceSensitivityDataJoinBolt(config);
    BoltDeclarer boltDeclarer = builder.setBolt("joinBolt", joinBolt, numOfJoinTasks);
    boltDeclarer.fieldsGrouping("parserBolt", new Fields("f1"));

    StormStreamSink sinkBolt = environment.getStreamSink("oozie_audit_log_stream", config);
    int numOfSinkTasks = config.getInt(SINK_TASK_NUM);
    BoltDeclarer kafkaBoltDeclarer = builder.setBolt("kafkaSink", sinkBolt, numOfSinkTasks);
    kafkaBoltDeclarer.fieldsGrouping("joinBolt", new Fields("user"));
    return builder.createTopology();
}
 

@Override
public StormTopology execute(Config config, StormEnvironment environment) {
    String deserClsName = config.getString("dataSourceConfig.deserializerClass");
    KafkaSourcedSpoutScheme scheme = new KafkaSourcedSpoutScheme(deserClsName, config);

    TopologyBuilder builder = new TopologyBuilder();
    BaseRichSpout spout1 = new KafkaOffsetSourceSpoutProvider().getSpout(config);
    BaseRichSpout spout2 = KafkaSourcedSpoutProvider.getSpout(config, scheme);

    int numOfSpoutTasks = config.getInt(SPOUT_TASK_NUM);
    int numOfDistributionTasks = config.getInt(DISTRIBUTION_TASK_NUM);

    builder.setSpout("kafkaLogLagChecker", spout1, numOfSpoutTasks);
    builder.setSpout("kafkaMessageFetcher", spout2, numOfSpoutTasks);

    KafkaMessageDistributionBolt bolt = new KafkaMessageDistributionBolt(config);
    BoltDeclarer bolteclarer = builder.setBolt("distributionBolt", bolt, numOfDistributionTasks);
    bolteclarer.fieldsGrouping("kafkaLogLagChecker", new Fields("f1"));
    bolteclarer.fieldsGrouping("kafkaLogLagChecker", new Fields("f1"));
    return builder.createTopology();
}
 

public static void main(String[] args) throws Exception{
    System.setProperty("config.resource", "/application.conf");
    Config config = ConfigFactory.load();
    KafkaSpoutProvider provider = new KafkaSpoutProvider();
    IRichSpout spout = provider.getSpout(config);

    SecurityLogParserBolt bolt = new SecurityLogParserBolt();
    TopologyBuilder builder = new TopologyBuilder();

    int numOfSpoutTasks = config.getInt(SPOUT_TASK_NUM);
    int numOfParserTasks = config.getInt(PARSER_TASK_NUM);
    int numOfSinkTasks = config.getInt(SINK_TASK_NUM);

    builder.setSpout("ingest", spout, numOfSpoutTasks);
    BoltDeclarer boltDeclarer = builder.setBolt("parserBolt", bolt, numOfParserTasks);
    boltDeclarer.shuffleGrouping("ingest");

    KafkaBolt kafkaBolt = new KafkaBolt();
    BoltDeclarer kafkaBoltDeclarer = builder.setBolt("kafkaSink", kafkaBolt, numOfSinkTasks);
    kafkaBoltDeclarer.shuffleGrouping("parserBolt");

    StormTopology topology = builder.createTopology();

    TopologySubmitter.submit(topology, config);
}
 

public static void SetRemoteTopology()
        throws Exception {
    String streamName = (String) conf.get(Config.TOPOLOGY_NAME);
    if (streamName == null) {
        String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\.");
        streamName = className[className.length - 1];
    }
    
    TopologyBuilder builder = new TopologyBuilder();
    
    int spout_Parallelism_hint = Utils.getInt(conf.get(TOPOLOGY_SPOUT_PARALLELISM_HINT), 1);
    int bolt_Parallelism_hint = Utils.getInt(conf.get(TOPOLOGY_BOLT_PARALLELISM_HINT), 2);
    builder.setSpout("spout", new TestSpout(), spout_Parallelism_hint);
    
    BoltDeclarer boltDeclarer = builder.setBolt("bolt", new TestBolt(), bolt_Parallelism_hint);
    // localFirstGrouping is only for jstorm
    // boltDeclarer.localFirstGrouping(SequenceTopologyDef.SEQUENCE_SPOUT_NAME);
    boltDeclarer.shuffleGrouping("spout");
    // .addConfiguration(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS, 60);
    
    StormSubmitter.submitTopology(streamName, conf, builder.createTopology());
    
}
 

public static void SetRemoteTopology()
        throws Exception {
    String streamName = (String) conf.get(Config.TOPOLOGY_NAME);
    if (streamName == null) {
        String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\.");
        streamName = className[className.length - 1];
    }
    
    TopologyBuilder builder = new TopologyBuilder();
    
    int spout_Parallelism_hint = Utils.getInt(conf.get(TOPOLOGY_SPOUT_PARALLELISM_HINT), 1);
    int bolt_Parallelism_hint = Utils.getInt(conf.get(TOPOLOGY_BOLT_PARALLELISM_HINT), 2);
    builder.setSpout("spout", new TestSpout(), spout_Parallelism_hint);
    
    BoltDeclarer boltDeclarer = builder.setBolt("bolt", new TestBolt(), bolt_Parallelism_hint);
    // localFirstGrouping is only for jstorm
    // boltDeclarer.localFirstGrouping(SequenceTopologyDef.SEQUENCE_SPOUT_NAME);
    boltDeclarer.shuffleGrouping("spout");
    // .addConfiguration(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS, 60);
    
    StormSubmitter.submitTopology(streamName, conf, builder.createTopology());
    
}
 

@Override
public StormTopology execute(Config config, StormEnvironment environment) {
    TopologyBuilder builder = new TopologyBuilder();
    KafkaSpoutProvider provider = new KafkaSpoutProvider();
    IRichSpout spout = provider.getSpout(config);

    int numOfSpoutTasks = config.getInt(SPOUT_TASK_NUM);
    int numOfAnalyzerTasks = config.getInt(ANALYZER_TASK_NUM);
    int numOfGeneratorTasks = config.getInt(GENERATOR_TASK_NUM);
    int numOfSinkTasks = config.getInt(SINK_TASK_NUM);

    builder.setSpout("ingest", spout, numOfSpoutTasks);

    GCLogAnalyzerBolt bolt = new GCLogAnalyzerBolt();
    BoltDeclarer boltDeclarer = builder.setBolt("analyzerBolt", bolt, numOfAnalyzerTasks);
    boltDeclarer.fieldsGrouping("ingest", new Fields(StringScheme.STRING_SCHEME_KEY));

    GCMetricGeneratorBolt generatorBolt = new GCMetricGeneratorBolt(config);
    BoltDeclarer joinBoltDeclarer = builder.setBolt("generatorBolt", generatorBolt, numOfGeneratorTasks);
    joinBoltDeclarer.fieldsGrouping("analyzerBolt", new Fields("f1"));

    StormStreamSink sinkBolt = environment.getStreamSink("gc_log_stream",config);
    BoltDeclarer kafkaBoltDeclarer = builder.setBolt("kafkaSink", sinkBolt, numOfSinkTasks);
    kafkaBoltDeclarer.fieldsGrouping("generatorBolt", new Fields("f1"));
    return builder.createTopology();
}
 

public static void SetRemoteTopology()
        throws AlreadyAliveException, InvalidTopologyException, TopologyAssignException {
    String streamName = (String) conf.get(Config.TOPOLOGY_NAME);
    if (streamName == null) {
        String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\.");
        streamName = className[className.length - 1];
    }
    
    TopologyBuilder builder = new TopologyBuilder();
    
    int spout_Parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_SPOUT_PARALLELISM_HINT), 1);
    int bolt_Parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_BOLT_PARALLELISM_HINT), 2);
    builder.setSpout("spout", new TestSpout(), spout_Parallelism_hint);
    
    BoltDeclarer boltDeclarer = builder.setBolt("bolt", new TestBolt(), bolt_Parallelism_hint);
    // localFirstGrouping is only for jstorm
    // boltDeclarer.localFirstGrouping(SequenceTopologyDef.SEQUENCE_SPOUT_NAME);
    boltDeclarer.shuffleGrouping("spout");
    // .addConfiguration(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS, 60);
    
    conf.put(Config.STORM_CLUSTER_MODE, "distributed");
    
    StormSubmitter.submitTopology(streamName, conf, builder.createTopology());
    
}
 

@Override
public StormTopology execute(Config config, StormEnvironment environment) {
    TopologyBuilder builder = new TopologyBuilder();
    KafkaSpoutProvider provider = new KafkaSpoutProvider();
    IRichSpout spout = provider.getSpout(config);

    HBaseAuditLogParserBolt bolt = new HBaseAuditLogParserBolt();

    int numOfSpoutTasks = config.getInt(SPOUT_TASK_NUM);
    int numOfParserTasks = config.getInt(PARSER_TASK_NUM);
    int numOfJoinTasks = config.getInt(JOIN_TASK_NUM);
    int numOfSinkTasks = config.getInt(SINK_TASK_NUM);

    builder.setSpout("ingest", spout, numOfSpoutTasks);
    BoltDeclarer boltDeclarer = builder.setBolt("parserBolt", bolt, numOfParserTasks);
    boltDeclarer.fieldsGrouping("ingest", new Fields(StringScheme.STRING_SCHEME_KEY));

    HBaseResourceSensitivityDataJoinBolt joinBolt = new HBaseResourceSensitivityDataJoinBolt(config);
    BoltDeclarer joinBoltDeclarer = builder.setBolt("joinBolt", joinBolt, numOfJoinTasks);
    joinBoltDeclarer.fieldsGrouping("parserBolt", new Fields("f1"));

    StormStreamSink sinkBolt = environment.getStreamSink("hbase_audit_log_stream",config);
    BoltDeclarer kafkaBoltDeclarer = builder.setBolt("kafkaSink", sinkBolt, numOfSinkTasks);
    kafkaBoltDeclarer.fieldsGrouping("joinBolt", new Fields("user"));
    return builder.createTopology();
}
 

@Override
public void addToTopology(StormTopology topology, int parallelismHint) {
  if (piBoltDeclarer != null) {
    // throw exception that one PI only belong to one topology
  } else {
    TopologyBuilder stormBuilder = topology.getStormBuilder();
    this.piBoltDeclarer = stormBuilder.setBolt(this.getName(),
        this.piBolt, parallelismHint);
  }
}
 

public static void main(String[] args) throws AlreadyAliveException, InvalidTopologyException {
    Map config = new Config();
    config.put(ConfigExtension.TOPOLOGY_MASTER_USER_DEFINED_STREAM_CLASS, "com.alipay.dw.jstorm.example.tm.TMUdfHandler");
    config.put(Config.TOPOLOGY_WORKERS, 2);

    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("TMUdfSpout", new TMUdfSpout(), 2);
    builder.setBolt("TMUdfBolt", new TMUdfBolt(), 4);
    StormTopology topology = builder.createTopology();

    StormSubmitter.submitTopology("TMUdfTopology", config, topology);
}
 

@Override
public void addToTopology(StormTopology topology, int parallelismHint) {
	if(piBoltDeclarer != null){
		//throw exception that one PI only belong to one topology
	}else{		
		TopologyBuilder stormBuilder = topology.getStormBuilder();
		this.piBoltDeclarer = stormBuilder.setBolt(this.getName(), 
				this.piBolt, parallelismHint);
	}	
}
 

/**
 * @return A topology builder than can further be customized.
 */
public TopologyBuilder create() {

    TopologyBuilder builder = new TopologyBuilder();

    if(eventsComponent instanceof IRichSpout)
      builder.setSpout(EVENT, (IRichSpout) eventsComponent, eventLoaderParallelism == -1 ? parallelismHint : eventLoaderParallelism);
    else if(eventsComponent instanceof IRichBolt)
      builder.setBolt(EVENT, (IRichBolt) eventsComponent, eventLoaderParallelism == -1 ? parallelismHint : eventLoaderParallelism);
    else
      throw new RuntimeException("The component for events is not valid. Must be IRichSpout or IRichBolt");


    if(flowLoaderSpout instanceof IRichSpout)
      builder.setSpout(FLOW_LOADER_STREAM, (IRichSpout) flowLoaderSpout, 1);
    else if(flowLoaderSpout instanceof IRichBolt)
      builder.setBolt(FLOW_LOADER_STREAM, (IRichBolt) flowLoaderSpout, 1);
    else
      throw new RuntimeException("The component for rules is not valid. Must be IRichSpout or IRichBolt");

    builder.setSpout("tick", new TickSpout(1000), 1);
    builder.setBolt(INITIALIZER, new FlowInitializerBolt(), parallelismHint)  // kicks off a flow determining where to start
            .localOrShuffleGrouping(EVENT)
            .allGrouping(FLOW_LOADER_STREAM, FLOW_LOADER_STREAM);

    declarebolt(builder, FILTER, new FilterBolt(), parallelismHint, true);
    declarebolt(builder, SELECT, new SelectorBolt(), parallelismHint, true);
    declarebolt(builder, PARTITION, new PartitionBolt(), parallelismHint, true);
    declarebolt(builder, SWITCH, new SwitchBolt(), parallelismHint, true);
    declarebolt(builder, AGGREGATE, new AggregatorBolt(), parallelismHint, true);
    declarebolt(builder, JOIN, new JoinBolt(), parallelismHint, true);
    declarebolt(builder, EACH, new EachBolt(), parallelismHint, true);
    declarebolt(builder, SORT, new SortBolt(), parallelismHint, true);
    declarebolt(builder, SPLIT, new SplitBolt(), parallelismHint, true);
    declarebolt(builder, OUTPUT, outputBolt, parallelismHint, false);

    return builder;
}
 

@Override
public StormTopology execute(Config config, StormEnvironment environment) {
    TopologyBuilder builder = new TopologyBuilder();
    HiveJobRunningSourcedStormSpoutProvider provider = new HiveJobRunningSourcedStormSpoutProvider();
    IRichSpout spout = provider.getSpout(config, config.getInt(SPOUT_TASK_NUM));


    int numOfSpoutTasks = config.getInt(SPOUT_TASK_NUM);
    int numOfFilterTasks = config.getInt(FILTER_TASK_NUM);
    int numOfParserTasks = config.getInt(PARSER_TASK_NUM);
    int numOfJoinTasks = config.getInt(JOIN_TASK_NUM);
    int numOfSinkTasks = config.getInt(SINK_TASK_NUM);

    builder.setSpout("ingest", spout, numOfSpoutTasks);
    JobFilterBolt bolt = new JobFilterBolt();
    BoltDeclarer boltDeclarer = builder.setBolt("filterBolt", bolt, numOfFilterTasks);
    boltDeclarer.fieldsGrouping("ingest", new Fields("jobId"));

    HiveQueryParserBolt parserBolt = new HiveQueryParserBolt();
    BoltDeclarer parserBoltDeclarer = builder.setBolt("parserBolt", parserBolt, numOfParserTasks);
    parserBoltDeclarer.fieldsGrouping("filterBolt", new Fields("user"));

    HiveSensitivityDataEnrichBolt joinBolt = new HiveSensitivityDataEnrichBolt(config);
    BoltDeclarer joinBoltDeclarer = builder.setBolt("joinBolt", joinBolt, numOfJoinTasks);
    joinBoltDeclarer.fieldsGrouping("parserBolt", new Fields("user"));

    StormStreamSink sinkBolt = environment.getStreamSink("hive_query_stream", config);
    BoltDeclarer kafkaBoltDeclarer = builder.setBolt("kafkaSink", sinkBolt, numOfSinkTasks);
    kafkaBoltDeclarer.fieldsGrouping("joinBolt", new Fields("user"));
    return builder.createTopology();
}
 

@Ignore  // this test need zookeeper
@Test
public void testConfigNotify() throws Exception {
    final String topoId = "myTopology";
    int numGroupbyBolts = 2;
    int numTotalGroupbyBolts = 3;
    System.setProperty("correlation.serviceHost", "sandbox.hortonworks.com");
    System.setProperty("correlation.servicePort", "58080");
    System.setProperty("withMetadataChangeNotifyService.zkConfig.zkQuorum", "sandbox.hortonworks.com:2181");
    System.setProperty("correlation.numGroupbyBolts", String.valueOf(numGroupbyBolts));
    System.setProperty("correlation.topologyName", topoId);
    System.setProperty("correlation.mode", "local");
    System.setProperty("correlation.zkHosts", "sandbox.hortonworks.com:2181");
    final String topicName1 = "testTopic3";
    final String topicName2 = "testTopic4";
    // ensure topic ready
    LogManager.getLogger(CorrelationSpout.class).setLevel(Level.DEBUG);
    Config config = ConfigFactory.load();

    CreateTopicUtils.ensureTopicReady(System.getProperty("withMetadataChangeNotifyService.zkConfig.zkQuorum"), topicName1);
    CreateTopicUtils.ensureTopicReady(System.getProperty("withMetadataChangeNotifyService.zkConfig.zkQuorum"), topicName2);

    TopologyBuilder topoBuilder = new TopologyBuilder();

    int numBolts = config.getInt("correlation.numGroupbyBolts");
    String spoutId = "correlation-spout";
    CorrelationSpout spout = new CorrelationSpout(config, topoId,
        new MockMetadataChangeNotifyService(topoId, spoutId), numBolts);
    SpoutDeclarer declarer = topoBuilder.setSpout(spoutId, spout);
    declarer.setNumTasks(2);
    for (int i = 0; i < numBolts; i++) {
        TestBolt bolt = new TestBolt();
        BoltDeclarer boltDecl = topoBuilder.setBolt("engineBolt" + i, bolt);
        boltDecl.fieldsGrouping(spoutId,
            StreamIdConversion.generateStreamIdBetween(AlertConstants.DEFAULT_SPOUT_NAME, AlertConstants.DEFAULT_ROUTERBOLT_NAME + i), new Fields());
    }

    String topoName = config.getString("correlation.topologyName");
    LOG.info("start topology in local mode");
    LocalCluster cluster = new LocalCluster();
    StormTopology topology = topoBuilder.createTopology();
    cluster.submitTopology(topoName, new HashMap(), topology);


    Utils.sleep(Long.MAX_VALUE);
}
 

@Ignore
@Test
public void testMultipleTopics() throws Exception {
    final String topoId = "myTopology";
    int numGroupbyBolts = 2;
    int numTotalGroupbyBolts = 3;
    System.setProperty("eagle.correlation.numGroupbyBolts", String.valueOf(numGroupbyBolts));
    System.setProperty("eagle.correlation.topologyName", topoId);
    System.setProperty("eagle.correlation.mode", "local");
    System.setProperty("eagle.correlation.zkHosts", "localhost:2181");
    final String topicName1 = "testTopic3";
    final String topicName2 = "testTopic4";
    // ensure topic ready
    LogManager.getLogger(CorrelationSpout.class).setLevel(Level.DEBUG);
    Config config = ConfigFactory.load();

    CreateTopicUtils.ensureTopicReady(System.getProperty("eagle.correlation.zkHosts"), topicName1);
    CreateTopicUtils.ensureTopicReady(System.getProperty("eagle.correlation.zkHosts"), topicName2);

    TopologyBuilder topoBuilder = new TopologyBuilder();

    int numBolts = config.getInt("eagle.correlation.numGroupbyBolts");
    CorrelationSpout spout = new CorrelationSpout(config, topoId, null, numBolts);
    String spoutId = "correlation-spout";
    SpoutDeclarer declarer = topoBuilder.setSpout(spoutId, spout);
    for (int i = 0; i < numBolts; i++) {
        TestBolt bolt = new TestBolt();
        BoltDeclarer boltDecl = topoBuilder.setBolt("engineBolt" + i, bolt);
        boltDecl.fieldsGrouping(spoutId, "stream_" + i, new Fields());
    }

    String topoName = config.getString("eagle.correlation.topologyName");
    LOG.info("start topology in local mode");
    LocalCluster cluster = new LocalCluster();
    cluster.submitTopology(topoName, new HashMap<>(), topoBuilder.createTopology());

    while (true) {
        try {
            Thread.sleep(1000);
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}
 

public static void SetBuilder(TopologyBuilder builder, Map conf) {
    
    int spout_Parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_SPOUT_PARALLELISM_HINT), 1);
    int bolt_Parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_BOLT_PARALLELISM_HINT), 2);
    
    builder.setSpout(SequenceTopologyDef.SEQUENCE_SPOUT_NAME, new SequenceSpout(), spout_Parallelism_hint);
    
    boolean isEnableSplit = JStormUtils.parseBoolean(conf.get("enable.split"), false);
    
    if (!isEnableSplit) {
        BoltDeclarer boltDeclarer = builder.setBolt(SequenceTopologyDef.TOTAL_BOLT_NAME, new TotalCount(),
                bolt_Parallelism_hint);
                
        // localFirstGrouping is only for jstorm
        // boltDeclarer.localFirstGrouping(SequenceTopologyDef.SEQUENCE_SPOUT_NAME);
        boltDeclarer.shuffleGrouping(SequenceTopologyDef.SEQUENCE_SPOUT_NAME)
                .allGrouping(SequenceTopologyDef.SEQUENCE_SPOUT_NAME, SequenceTopologyDef.CONTROL_STREAM_ID)
                .addConfiguration(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS, 3);
    } else {
        
        builder.setBolt(SequenceTopologyDef.SPLIT_BOLT_NAME, new SplitRecord(), bolt_Parallelism_hint)
                .localOrShuffleGrouping(SequenceTopologyDef.SEQUENCE_SPOUT_NAME);
                
        builder.setBolt(SequenceTopologyDef.TRADE_BOLT_NAME, new PairCount(), bolt_Parallelism_hint)
                .shuffleGrouping(SequenceTopologyDef.SPLIT_BOLT_NAME, SequenceTopologyDef.TRADE_STREAM_ID);
        builder.setBolt(SequenceTopologyDef.CUSTOMER_BOLT_NAME, new PairCount(), bolt_Parallelism_hint)
                .shuffleGrouping(SequenceTopologyDef.SPLIT_BOLT_NAME, SequenceTopologyDef.CUSTOMER_STREAM_ID);
                
        builder.setBolt(SequenceTopologyDef.MERGE_BOLT_NAME, new MergeRecord(), bolt_Parallelism_hint)
                .fieldsGrouping(SequenceTopologyDef.TRADE_BOLT_NAME, new Fields("ID"))
                .fieldsGrouping(SequenceTopologyDef.CUSTOMER_BOLT_NAME, new Fields("ID"));
                
        builder.setBolt(SequenceTopologyDef.TOTAL_BOLT_NAME, new TotalCount(), bolt_Parallelism_hint)
                .noneGrouping(SequenceTopologyDef.MERGE_BOLT_NAME);
    }
    
    boolean kryoEnable = JStormUtils.parseBoolean(conf.get("kryo.enable"), false);
    if (kryoEnable) {
        System.out.println("Use Kryo ");
        boolean useJavaSer = JStormUtils.parseBoolean(conf.get("fall.back.on.java.serialization"), true);
        
        Config.setFallBackOnJavaSerialization(conf, useJavaSer);
        
        Config.registerSerialization(conf, TradeCustomer.class, TradeCustomerSerializer.class);
        Config.registerSerialization(conf, Pair.class, PairSerializer.class);
    }
    
    // conf.put(Config.TOPOLOGY_DEBUG, false);
    // conf.put(ConfigExtension.TOPOLOGY_DEBUG_RECV_TUPLE, false);
    // conf.put(Config.STORM_LOCAL_MODE_ZMQ, false);
    
    int ackerNum = JStormUtils.parseInt(conf.get(Config.TOPOLOGY_ACKER_EXECUTORS), 1);
    Config.setNumAckers(conf, ackerNum);
    // conf.put(Config.TOPOLOGY_MAX_TASK_PARALLELISM, 6);
    // conf.put(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS, 20);
    // conf.put(Config.TOPOLOGY_MAX_SPOUT_PENDING, 1);
    
    int workerNum = JStormUtils.parseInt(conf.get(Config.TOPOLOGY_WORKERS), 20);
    conf.put(Config.TOPOLOGY_WORKERS, workerNum);
    
}
 

@Override
public StormTopology execute(Config config, StormEnvironment environment) {
    TopologyBuilder builder = new TopologyBuilder();
    KafkaSpoutProvider provider = new KafkaSpoutProvider();
    IRichSpout spout = provider.getSpout(config);

    int numOfSpoutTasks = config.getInt(SPOUT_TASK_NUM);
    int numOfParserTasks = config.getInt(PARSER_TASK_NUM);
    int numOfSensitivityJoinTasks = config.getInt(SENSITIVITY_JOIN_TASK_NUM);
    int numOfIPZoneJoinTasks = config.getInt(IPZONE_JOIN_TASK_NUM);
    int numOfSinkTasks = config.getInt(SINK_TASK_NUM);
    int numOfTrafficMonitorTasks = config.hasPath(TRAFFIC_MONITOR_TASK_NUM) ? config.getInt(TRAFFIC_MONITOR_TASK_NUM) : numOfParserTasks;

    builder.setSpout("ingest", spout, numOfSpoutTasks).setNumTasks(numOfSpoutTasks);

    // ---------------------
    // ingest -> parserBolt
    // ---------------------

    BaseRichBolt parserBolt = getParserBolt(config);
    BoltDeclarer boltDeclarer = builder.setBolt("parserBolt", parserBolt, numOfParserTasks).setNumTasks(numOfParserTasks).shuffleGrouping("ingest");
    boltDeclarer.shuffleGrouping("ingest");

    // Boolean useDefaultPartition = !config.hasPath("eagleProps.useDefaultPartition") || config.getBoolean("eagleProps.useDefaultPartition");
    // if (useDefaultPartition) {
    //    boltDeclarer.fieldsGrouping("ingest", new Fields(StringScheme.STRING_SCHEME_KEY));
    // } else {
    //    boltDeclarer.customGrouping("ingest", new CustomPartitionGrouping(createStrategy(config)));
    // }

    // ------------------------------
    // parserBolt -> sensitivityJoin
    // ------------------------------

    HdfsSensitivityDataEnrichBolt sensitivityDataJoinBolt = new HdfsSensitivityDataEnrichBolt(config);
    BoltDeclarer sensitivityDataJoinBoltDeclarer = builder.setBolt("sensitivityJoin", sensitivityDataJoinBolt, numOfSensitivityJoinTasks).setNumTasks(numOfSensitivityJoinTasks);
    // sensitivityDataJoinBoltDeclarer.fieldsGrouping("parserBolt", new Fields("f1"));
    sensitivityDataJoinBoltDeclarer.shuffleGrouping("parserBolt");

    if (config.hasPath(TRAFFIC_MONITOR_ENABLED) && config.getBoolean(TRAFFIC_MONITOR_ENABLED)) {
        HadoopLogAccumulatorBolt auditLogAccumulator = new HadoopLogAccumulatorBolt(config);
        BoltDeclarer auditLogAccumulatorDeclarer = builder.setBolt("logAccumulator", auditLogAccumulator, numOfTrafficMonitorTasks);
        auditLogAccumulatorDeclarer.setNumTasks(numOfTrafficMonitorTasks).shuffleGrouping("parserBolt");
    }

    // ------------------------------
    // sensitivityJoin -> ipZoneJoin
    // ------------------------------
    IPZoneDataEnrichBolt ipZoneDataJoinBolt = new IPZoneDataEnrichBolt(config);
    BoltDeclarer ipZoneDataJoinBoltDeclarer = builder.setBolt("ipZoneJoin", ipZoneDataJoinBolt, numOfIPZoneJoinTasks).setNumTasks(numOfIPZoneJoinTasks);
    // ipZoneDataJoinBoltDeclarer.fieldsGrouping("sensitivityJoin", new Fields("user"));
    ipZoneDataJoinBoltDeclarer.shuffleGrouping("sensitivityJoin");

    // ------------------------
    // ipZoneJoin -> kafkaSink
    // ------------------------

    StormStreamSink sinkBolt = environment.getStreamSink("hdfs_audit_log_stream", config);
    BoltDeclarer kafkaBoltDeclarer = builder.setBolt("kafkaSink", sinkBolt, numOfSinkTasks).setNumTasks(numOfSinkTasks);
    kafkaBoltDeclarer.shuffleGrouping("ipZoneJoin");
    return builder.createTopology();
}
 

private StormTopology createTopology(DRPCSpout spout) {
    final String SPOUT_ID = "spout";
    final String PREPARE_ID = "prepare-request";

    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout(SPOUT_ID, spout);
    builder.setBolt(PREPARE_ID, new PrepareRequest()).noneGrouping(SPOUT_ID);
    int i = 0;
    for (; i < _components.size(); i++) {
        Component component = _components.get(i);

        Map<String, SourceArgs> source = new HashMap<String, SourceArgs>();
        if (i == 1) {
            source.put(boltId(i - 1), SourceArgs.single());
        } else if (i >= 2) {
            source.put(boltId(i - 1), SourceArgs.all());
        }
        IdStreamSpec idSpec = null;
        if (i == _components.size() - 1 && component.bolt instanceof FinishedCallback) {
            idSpec = IdStreamSpec.makeDetectSpec(PREPARE_ID, PrepareRequest.ID_STREAM);
        }
        BoltDeclarer declarer = builder.setBolt(boltId(i), new CoordinatedBolt(component.bolt, source, idSpec), component.parallelism);

        for (Map conf : component.componentConfs) {
            declarer.addConfigurations(conf);
        }

        if (idSpec != null) {
            declarer.fieldsGrouping(idSpec.getGlobalStreamId().get_componentId(), PrepareRequest.ID_STREAM, new Fields("request"));
        }
        if (i == 0 && component.declarations.isEmpty()) {
            declarer.noneGrouping(PREPARE_ID, PrepareRequest.ARGS_STREAM);
        } else {
            String prevId;
            if (i == 0) {
                prevId = PREPARE_ID;
            } else {
                prevId = boltId(i - 1);
            }
            for (InputDeclaration declaration : component.declarations) {
                declaration.declare(prevId, declarer);
            }
        }
        if (i > 0) {
            declarer.directGrouping(boltId(i - 1), Constants.COORDINATED_STREAM_ID);
        }
    }

    IRichBolt lastBolt = _components.get(_components.size() - 1).bolt;
    OutputFieldsGetter getter = new OutputFieldsGetter();
    lastBolt.declareOutputFields(getter);
    Map<String, StreamInfo> streams = getter.getFieldsDeclaration();
    if (streams.size() != 1) {
        throw new RuntimeException("Must declare exactly one stream from last bolt in LinearDRPCTopology");
    }
    String outputStream = streams.keySet().iterator().next();
    List<String> fields = streams.get(outputStream).get_output_fields();
    if (fields.size() != 2) {
        throw new RuntimeException(
                "Output stream of last component in LinearDRPCTopology must contain exactly two fields. The first should be the request id, and the second should be the result.");
    }

    builder.setBolt("JoinResult", new JoinResult(PREPARE_ID)).fieldsGrouping(boltId(i - 1), outputStream, new Fields(fields.get(0)))
            .fieldsGrouping(PREPARE_ID, PrepareRequest.RETURN_STREAM, new Fields("request"));
    i++;
    builder.setBolt("ReturnResults", new ReturnResults()).noneGrouping("JoinResult");
    return builder.createTopology();
}
 

public static void main(String[] args) throws Exception {
    TopologyBuilder builder = new TopologyBuilder();
    
    builder.setBolt("ClusterInfo", new ClusterInfoBolt(), 1);
    Config conf = new Config();
    conf.setNumWorkers(1);
    
    StormSubmitter.submitTopology("ClusterMonitor", conf, builder.createTopology());

}