Java Code Examples for org.apache.storm.Config#setNumWorkers()

public static void main(String[] args) throws Exception {

        TopologyBuilder builder = new TopologyBuilder();

        builder.setSpout("spout", new RandomSentence(), 5);

        builder.setBolt("split", new SplitSentence(), 8).shuffleGrouping("spout");
        builder.setBolt("count", new WordCount(), 12).fieldsGrouping("split", new Fields("word"));

        Config conf = new Config();
        conf.setDebug(true);
        String topoName = "word-count";
        if (args != null && args.length > 0) {
            topoName = args[0];
        }
        conf.setNumWorkers(3);
        StormSubmitter.submitTopologyWithProgressBar(topoName, conf, builder.createTopology());
    }
 

public static void main(String[] args) throws Exception {
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("integer", new RandomIntegerSpout(), 1);
    builder.setBolt("slidingsum", new SlidingWindowSumBolt().withWindow(Count.of(30), Count.of(10)), 1)
           .shuffleGrouping("integer");
    builder.setBolt("tumblingavg", new TumblingWindowAvgBolt().withTumblingWindow(Count.of(3)), 1)
           .shuffleGrouping("slidingsum");
    builder.setBolt("printer", new PrinterBolt(), 1).shuffleGrouping("tumblingavg");
    Config conf = new Config();
    conf.setDebug(true);
    String topoName = "test";
    if (args != null && args.length > 0) {
        topoName = args[0];
    }
    conf.setNumWorkers(1);
    StormSubmitter.submitTopologyWithProgressBar(topoName, conf, builder.createTopology());
}
 

public static void main(String[] args) throws Exception {
    TopologyBuilder builder = new TopologyBuilder();

    builder.setSpout("word", new TestWordSpout(), 10);
    builder.setBolt("exclaim1", new ExclamationLoggingBolt(), 3).shuffleGrouping("word");
    builder.setBolt("exclaim2", new ExclamationLoggingBolt(), 2).shuffleGrouping("exclaim1");

    Config conf = new Config();
    conf.setDebug(true);
    String topoName = MultipleLoggerTopology.class.getName();
    if (args != null && args.length > 0) {
        topoName = args[0];
    }
    conf.setNumWorkers(2);
    StormSubmitter.submitTopologyWithProgressBar(topoName, conf, builder.createTopology());
}
 

public static void main(String[] args) throws Exception {
    TopologyBuilder builder = new TopologyBuilder();
    BaseWindowedBolt bolt = new SlidingWindowSumBolt()
        .withWindow(new Duration(5, TimeUnit.SECONDS), new Duration(3, TimeUnit.SECONDS))
        .withTimestampField("ts")
        .withLag(new Duration(5, TimeUnit.SECONDS));
    builder.setSpout("integer", new RandomIntegerSpout(), 1);
    builder.setBolt("slidingsum", bolt, 1).shuffleGrouping("integer");
    builder.setBolt("printer", new PrinterBolt(), 1).shuffleGrouping("slidingsum");
    Config conf = new Config();
    conf.setDebug(true);
    String topoName = "test";

    if (args != null && args.length > 0) {
        topoName = args[0];
    }

    conf.setNumWorkers(1);
    StormSubmitter.submitTopologyWithProgressBar(topoName, conf, builder.createTopology());
}
 

public static void main(String[] args) throws Exception {
    StreamBuilder builder = new StreamBuilder();
    /**
     * The spout emits sequences of (Integer, Long, Long). TupleValueMapper can be used to extract fields
     * from the values and produce a stream of typed tuple (Tuple3<Integer, Long, Long> in this case.
     */
    Stream<Tuple3<Integer, Long, Long>> stream = builder.newStream(new RandomIntegerSpout(), TupleValueMappers.of(0, 1, 2));

    PairStream<Long, Integer> pairs = stream.mapToPair(t -> Pair.of(t._2 / 10000, t._1));

    pairs.window(TumblingWindows.of(Count.of(10))).groupByKey().print();

    String topoName = "test";
    if (args.length > 0) {
        topoName = args[0];
    }
    Config config = new Config();
    config.setNumWorkers(1);
    StormSubmitter.submitTopologyWithProgressBar(topoName, config, builder.build());
}
 

private void start(StormTopology topology, boolean runAsLocal) throws Exception {
    Config conf = new Config();
    conf.put(Constants.ZOOKEEPER_SERVERS, zkConnect);
    conf.put(Constants.TOPOLOGY_ID, topologyId);
    conf.put(Constants.SINK_TYPE, sinkType);
    conf.put(Config.TOPOLOGY_WORKER_CHILDOPTS, sinkerConf.getProperty(SinkerConstants.TOPOLOGY_WORKER_CHILDOPTS));

    conf.setMessageTimeoutSecs(Integer.parseInt(sinkerConf.getProperty(SinkerConstants.STORM_MESSAGE_TIMEOUT)));
    conf.setMaxSpoutPending(Integer.parseInt(sinkerConf.getProperty(SinkerConstants.STORM_MAX_SPOUT_PENDING)));
    conf.setDebug(true);
    conf.setNumWorkers(Integer.parseInt(sinkerConf.getProperty(SinkerConstants.STORM_NUM_WORKERS)));
    if (runAsLocal) {
        conf.setMaxTaskParallelism(10);
        LocalCluster cluster = new LocalCluster();
        cluster.submitTopology(topologyName, conf, topology);
    } else {
        StormSubmitter.submitTopology(topologyName, conf, topology);
    }
}
 

private StormTopology buildTopology() throws Exception {
    loadSinkerConf();

    Integer spoutSize = Integer.parseInt(sinkerConf.getProperty(SinkerConstants.STORM_KAFKA_READ_SPOUT_PARALLEL));
    Integer boltSize = Integer.parseInt(sinkerConf.getProperty(SinkerConstants.STORM_WRITE_BOUT_PARALLEL));
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("SinkerKafkaReadSpout", new SinkerKafkaReadSpout(), spoutSize);
    builder.setBolt("SinkerWriteBolt", new SinkerWriteBolt(), boltSize)
            .fieldsGrouping("SinkerKafkaReadSpout", "dataStream", new Fields("ns"))
            .allGrouping("SinkerKafkaReadSpout", "ctrlStream");

    Config conf = new Config();
    conf.setDebug(true);
    conf.setNumWorkers(1);

    return builder.createTopology();
}
 

@SuppressWarnings("unchecked")
public static void main(String[] args) throws Exception {
    StreamBuilder builder = new StreamBuilder();
    /**
     * Computes average of the stream of numbers emitted by the spout. Internally the per-partition
     * sum and counts are accumulated and emitted to a downstream task where the partially accumulated
     * results are merged and the final result is emitted.
     */
    builder.newStream(new RandomIntegerSpout(), new ValueMapper<Integer>(0), 2)
           .window(TumblingWindows.of(BaseWindowedBolt.Duration.seconds(5)))
           .filter(x -> x > 0 && x < 500)
           .aggregate(new Avg())
           .print();

    Config config = new Config();
    String topoName = "AGG_EXAMPLE";
    if (args.length > 0) {
        topoName = args[0];
    }
    config.setNumWorkers(1);
    StormSubmitter.submitTopologyWithProgressBar(topoName, config, builder.build());
}
 

private void start(StormTopology topology, boolean runAsLocal) throws Exception {
    Config conf = new Config();
    conf.put(com.creditease.dbus.commons.Constants.ZOOKEEPER_SERVERS, zkConnect);
    conf.put(Constants.TOPOLOGY_ID, topologyId);
    conf.put(Constants.TOPOLOGY_ALIAS, alias);
    conf.put(Constants.ROUTER_PROJECT_NAME, projectName);

    String workerChildOpts = routerConf.getProperty(DBusRouterConstants.STORM_TOPOLOGY_WORKER_CHILDOPTS, "-Xmx2g");
    conf.put(Config.TOPOLOGY_WORKER_CHILDOPTS, workerChildOpts);

    int msgTimeout = Integer.valueOf(routerConf.getProperty(DBusRouterConstants.STORM_MESSAGE_TIMEOUT, "10"));
    conf.setMessageTimeoutSecs(msgTimeout);

    int maxSpoutPending = Integer.valueOf(routerConf.getProperty(DBusRouterConstants.STORM_MAX_SPOUT_PENDING, "100"));
    conf.setMaxSpoutPending(maxSpoutPending);
    conf.setDebug(true);

    int numWorks = Integer.valueOf(routerConf.getProperty(DBusRouterConstants.STORM_NUM_WORKS, "1"));
    conf.setNumWorkers(numWorks);

    if (runAsLocal) {
        conf.setMaxTaskParallelism(10);
        LocalCluster cluster = new LocalCluster();
        cluster.submitTopology(topologyName, conf, topology);
    } else {
        StormSubmitter.submitTopology(topologyName, conf, topology);
    }
}
 

public static void main(String[] args) throws Exception {
    StreamBuilder builder = new StreamBuilder();
    StreamState<String, Long> ss = builder.newStream(new TestWordSpout(), new ValueMapper<String>(0), 2)
                                          /*
                                           * Transform the stream of words to a stream of (word, 1) pairs
                                           */
                                          .mapToPair(w -> Pair.of(w, 1))
                                          /*
                                           *  Update the count in the state. Here the first argument 0L is the initial value for the
                                           *  count and
                                           *  the second argument is a function that increments the count for each value received.
                                           */
                                          .updateStateByKey(0L, (count, val) -> count + 1);

    /*
     * A stream of words emitted by the QuerySpout is used as
     * the keys to query the state.
     */
    builder.newStream(new QuerySpout(), new ValueMapper<String>(0))
           /*
            * Queries the state and emits the
            * matching (key, value) as results. The stream state returned
            * by the updateStateByKey is passed as the argument to stateQuery.
            */
           .stateQuery(ss).print();

    Config config = new Config();
    // use redis based state store for persistence
    config.put(Config.TOPOLOGY_STATE_PROVIDER, "org.apache.storm.redis.state.RedisKeyValueStateProvider");
    String topoName = "test";
    if (args.length > 0) {
        topoName = args[0];
    }
    config.setNumWorkers(1);
    StormSubmitter.submitTopologyWithProgressBar(topoName, config, builder.build());
}
 

public static void main(String[] args) throws Exception {
    Config conf = new Config();
    WindowsStoreFactory mapState = new InMemoryWindowsStoreFactory();
    String topoName = "wordCounter";
    if (args.length > 0) {
        topoName = args[0];
    }

    conf.setNumWorkers(3);
    StormSubmitter.submitTopologyWithProgressBar(topoName, conf, buildTopology(mapState, SlidingCountWindow.of(1000, 100)));
}
 

public static void main(String[] args) throws Exception {
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("spout", new RandomIntegerSpout());
    builder.setBolt("partialsum", new StatefulSumBolt("partial"), 1).shuffleGrouping("spout");
    builder.setBolt("printer", new PrinterBolt(), 2).shuffleGrouping("partialsum");
    builder.setBolt("total", new StatefulSumBolt("total"), 1).shuffleGrouping("printer");
    Config conf = new Config();
    conf.setDebug(false);
    String topoName = "test";
    if (args != null && args.length > 0) {
        topoName = args[0];
    }
    conf.setNumWorkers(1);
    StormSubmitter.submitTopologyWithProgressBar(topoName, conf, builder.createTopology());
}
 

public void buildTopology(String[] args) {
    //TODO
    if (parseCommandArgs(args) != 0) {
        return;
    }
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("CanalClientSpout", new CanalClientSpout(), 1);
    builder.setBolt("KafkaProducerBolt", new KafkaProducerBolt(), 1).shuffleGrouping("CanalClientSpout");

    Config conf = new Config();
    conf.put(Constants.ZOOKEEPER_SERVERS, zkServers);
    conf.put(Constants.EXTRACTOR_TOPOLOGY_ID, extractorTopologyId);
    logger.info(Constants.ZOOKEEPER_SERVERS + "=" + zkServers);
    logger.info(Constants.EXTRACTOR_TOPOLOGY_ID + "=" + extractorTopologyId);
    conf.setNumWorkers(1);
    conf.setMaxSpoutPending(50);
    conf.setMessageTimeoutSecs(120);
    if (!runAsLocal) {
        conf.setDebug(false);
        try {
            //StormSubmitter.submitTopology("extractorTopologyId", conf, builder.createTopology());
            StormSubmitter.submitTopology(extractorTopologyId, conf, builder.createTopology());
        } catch (Exception e) {
            e.printStackTrace();
        }
    } else {
        conf.setDebug(false);
        LocalCluster cluster = new LocalCluster();
        //cluster.submitTopology("extractorTopologyId", conf, builder.createTopology());
        cluster.submitTopology(extractorTopologyId, conf, builder.createTopology());
    }
}
 

public static void main(String[] args) throws Exception {

        StormTopology topology = buildDevicesTopology();
        Config conf = new Config();
        conf.setMaxSpoutPending(20);
        conf.setNumWorkers(3);
        StormSubmitter.submitTopologyWithProgressBar("devices-topology", conf, topology);
    }
 

public static void main(String[] args) throws Exception {
  TopologyBuilder builder = new TopologyBuilder();
  int parallelism = 2;

  int spouts = parallelism;
  builder.setSpout("word", new TestWordSpout(Duration.ofMillis(50)), spouts);
  int bolts = 2 * parallelism;
  builder.setBolt("exclaim1", new ExclamationBolt(), bolts)
      .shuffleGrouping("word");

  Config conf = new Config();
  conf.setDebug(true);
  conf.setMaxSpoutPending(10);
  conf.setMessageTimeoutSecs(600);
  conf.put(Config.TOPOLOGY_WORKER_CHILDOPTS, "-XX:+HeapDumpOnOutOfMemoryError");

  if (args != null && args.length > 0) {
    conf.setNumWorkers(parallelism);
    StormSubmitter.submitTopology(args[0], conf, builder.createTopology());
  } else {
    System.out.println("Topology name not provided as an argument, running in simulator mode.");
    LocalCluster cluster = new LocalCluster();
    cluster.submitTopology("test", conf, builder.createTopology());
    Utils.sleep(10000);
    cluster.killTopology("test");
    cluster.shutdown();
  }
}
 

public  void runStorm(String[] args) {
	// 定义一个拓扑
	TopologyBuilder builder = new TopologyBuilder();
	// 设置1个Executeor(线程)，默认一个
	builder.setSpout(Constants.KAFKA_SPOUT, new KafkaInsertDataSpout(), 1);
	// shuffleGrouping:表示是随机分组
	// 设置1个Executeor(线程)，和两个task
	builder.setBolt(Constants.INSERT_BOLT, new InsertBolt(), 1).setNumTasks(1).shuffleGrouping(Constants.KAFKA_SPOUT);
	Config conf = new Config();
	//设置一个应答者
	conf.setNumAckers(1);
	//设置一个work
	conf.setNumWorkers(3);
	try {
		// 有参数时，表示向集群提交作业，并把第一个参数当做topology名称
		// 没有参数时，本地提交
		if (args != null && args.length > 0) { 
			logger.info("运行远程模式");
			StormSubmitter.submitTopology(args[0], conf, builder.createTopology());
		} else {
			// 启动本地模式
			logger.info("运行本地模式");
			LocalCluster cluster = new LocalCluster();
			cluster.submitTopology("TopologyApp", conf, builder.createTopology());
		}
	} catch (Exception e) {
		logger.error("storm启动失败!程序退出!",e);
		System.exit(1);
	}

	logger.info("storm启动成功...");

}
 

/**
 * Create and deploy the topology.
 *
 * @param args args
 * @throws Exception exception
 */
public static void main(String[] args) throws Exception {
    TopologyBuilder builder = new TopologyBuilder();

    // generate random numbers
    builder.setSpout("spout", new RandomIntegerSpout());

    // emits sliding window and global averages
    builder.setBolt("avgbolt", new AvgBolt()
        .withWindow(new Duration(10, TimeUnit.SECONDS), new Duration(2, TimeUnit.SECONDS))
        // persist the window in state
        .withPersistence()
        // max number of events to be cached in memory
        .withMaxEventsInMemory(25000), 1)
           .shuffleGrouping("spout");

    // print the values to stdout
    builder.setBolt("printer", (x, y) -> System.out.println(x.getValue(0)), 1).shuffleGrouping("avgbolt");

    Config conf = new Config();
    conf.setDebug(false);

    // checkpoint the state every 5 seconds
    conf.put(Config.TOPOLOGY_STATE_CHECKPOINT_INTERVAL, 5000);

    // use redis for state persistence
    conf.put(Config.TOPOLOGY_STATE_PROVIDER, "org.apache.storm.redis.state.RedisKeyValueStateProvider");

    String topoName = "test";
    if (args != null && args.length > 0) {
        topoName = args[0];
    }
    conf.setNumWorkers(1);
    StormSubmitter.submitTopologyWithProgressBar(topoName, conf, builder.createTopology());
}
 

private void start(StormTopology topology, boolean runAsLocal) throws Exception {

        Config conf = new Config();

        // 启动类型为all，或者dispatcher
        if (topologyType.equals(Constants.TopologyType.ALL) || topologyType.equals(Constants.TopologyType.DISPATCHER)) {
            /**
             * dispatcher配置
             */

            conf.put(com.creditease.dbus.commons.Constants.ZOOKEEPER_SERVERS, zookeeper);
            conf.put(com.creditease.dbus.commons.Constants.TOPOLOGY_ID, dispatcherTopologyId);
            logger.info(com.creditease.dbus.commons.Constants.ZOOKEEPER_SERVERS + "=" + zookeeper);
            logger.info(com.creditease.dbus.commons.Constants.TOPOLOGY_ID + "=" + dispatcherTopologyId);
        }

        // 启动类型为all，或者appender
        if (topologyType.equals(Constants.TopologyType.ALL) || topologyType.equals(Constants.TopologyType.APPENDER)) {

            /**
             * appender配置
             */
            conf.put(Constants.StormConfigKey.TOPOLOGY_ID, appenderTopologyId);
            conf.put(Constants.StormConfigKey.ZKCONNECT, zookeeper);
            conf.put(Constants.StormConfigKey.DATASOURCE, datasource);
        }

//        conf.put(Config.TOPOLOGY_TRANSFER_BUFFER_SIZE, 4096);
//        conf.put(Config.TOPOLOGY_EXECUTOR_RECEIVE_BUFFER_SIZE, 4096);
//        conf.put(Config.TOPOLOGY_EXECUTOR_SEND_BUFFER_SIZE, 4096);

        conf.setDebug(true);

        conf.setNumAckers(1);
        //设置worker数
        conf.setNumWorkers(1);
        //设置任务在发出后，但还没处理完成的中间状态任务的最大数量, 如果没有设置最大值为50
        int MaxSpoutPending = getConfigureValueWithDefault(Constants.ConfigureKey.MAX_SPOUT_PENDING, 50);
        conf.setMaxSpoutPending(MaxSpoutPending);
        //设置任务在多久之内没处理完成，则这个任务处理失败
        conf.setMessageTimeoutSecs(120);

        String opts = getWorkerChildopts();
        if (opts != null && opts.trim().length() > 0) {
            conf.put(Config.TOPOLOGY_WORKER_CHILDOPTS, opts);
        }

//        conf.put(Config.TOPOLOGY_SKIP_MISSING_KRYO_REGISTRATIONS, true);
//        conf.registerSerialization(org.apache.avro.util.Utf8.class);
//        conf.registerSerialization(com.creditease.dbus.commons.DBusConsumerRecord.class);
//        conf.registerSerialization(org.apache.kafka.common.record.TimestampType.class);
//        conf.registerSerialization(com.creditease.dbus.stream.common.appender.bean.EmitData.class);
//        conf.registerSerialization(com.creditease.dbus.stream.common.appender.enums.Command.class);
//        conf.registerSerialization(org.apache.avro.generic.GenericData.class);
//        conf.registerSerialization(com.creditease.dbus.stream.oracle.appender.avro.GenericData.class);
//        conf.registerSerialization(com.creditease.dbus.commons.DbusMessage12.class);
//        conf.registerSerialization(com.creditease.dbus.commons.DbusMessage12.Schema12.class);
//        conf.registerSerialization(com.creditease.dbus.commons.DbusMessage13.Schema13.class);
//        conf.registerSerialization(com.creditease.dbus.commons.DbusMessage13.class);
//        conf.registerSerialization(com.creditease.dbus.commons.DbusMessage.Field.class);
//        conf.registerSerialization(com.creditease.dbus.commons.DbusMessage.Payload.class);
//        conf.registerSerialization(com.creditease.dbus.commons.DbusMessage.Protocol.class);
//        conf.registerSerialization(com.creditease.dbus.commons.DbusMessage.ProtocolType.class);
//        conf.registerSerialization(com.creditease.dbus.stream.oracle.appender.bolt.processor.appender.OraWrapperData.class);
//        conf.registerSerialization(com.creditease.dbus.stream.common.appender.spout.cmds.TopicResumeCmd.class);

        if (runAsLocal) {
            LocalCluster cluster = new LocalCluster();
            cluster.submitTopology(topologyId, conf, topology);
            /*String cmd;
            do {
                cmd = System.console().readLine();
            } while (!cmd.equals("exit"));
            cluster.shutdown();*/
        } else {
            StormSubmitter.submitTopology(topologyId, conf, topology);
        }
    }
 

public static void main(String[] args) throws Exception {
    TopologyBuilder builder = new TopologyBuilder();

    Options opts = new Options();
    opts.addOption("conf", true, "Path to the config file.");

    CommandLineParser parser = new DefaultParser();
    CommandLine cmd = parser.parse(opts, args);
    String configPath = cmd.getOptionValue("conf");
    Map commonConfig = Utils.findAndReadConfigFile(configPath, true);

    String zkServerHosts = joinHosts((List<String>)commonConfig.get("zookeeper.servers"),
                                     Integer.toString((Integer)commonConfig.get("zookeeper.port")));
    String redisServerHost = (String)commonConfig.get("redis.host");
    String kafkaTopic = (String)commonConfig.get("kafka.topic");
    int kafkaPartitions = ((Number)commonConfig.get("kafka.partitions")).intValue();
    int workers = ((Number)commonConfig.get("storm.workers")).intValue();
    int ackers = ((Number)commonConfig.get("storm.ackers")).intValue();
    int cores = ((Number)commonConfig.get("process.cores")).intValue();
    int parallel = Math.max(1, cores/7);

    ZkHosts hosts = new ZkHosts(zkServerHosts);



    SpoutConfig spoutConfig = new SpoutConfig(hosts, kafkaTopic, "/" + kafkaTopic, UUID.randomUUID().toString());
    spoutConfig.scheme = new SchemeAsMultiScheme(new StringScheme());
    KafkaSpout kafkaSpout = new KafkaSpout(spoutConfig);

    builder.setSpout("ads", kafkaSpout, kafkaPartitions);
    builder.setBolt("event_deserializer", new DeserializeBolt(), parallel).shuffleGrouping("ads");
    builder.setBolt("event_filter", new EventFilterBolt(), parallel).shuffleGrouping("event_deserializer");
    builder.setBolt("event_projection", new EventProjectionBolt(), parallel).shuffleGrouping("event_filter");
    builder.setBolt("redis_join", new RedisJoinBolt(redisServerHost), parallel).shuffleGrouping("event_projection");
    builder.setBolt("campaign_processor", new CampaignProcessor(redisServerHost), parallel*2)
        .fieldsGrouping("redis_join", new Fields("campaign_id"));

    Config conf = new Config();

    if (args != null && args.length > 0) {
        conf.setNumWorkers(workers);
        conf.setNumAckers(ackers);
        StormSubmitter.submitTopologyWithProgressBar(args[0], conf, builder.createTopology());
    }
    else {

        LocalCluster cluster = new LocalCluster();
        cluster.submitTopology("test", conf, builder.createTopology());
        org.apache.storm.utils.Utils.sleep(10000);
        cluster.killTopology("test");
        cluster.shutdown();
    }
}
 

public static void main(String[] args) throws Exception {
  TopologyBuilder builder = new TopologyBuilder();

  builder.setSpout("word", new TestWordSpout(), 2);
  builder.setBolt("mybolt", new MyBolt(), 2)
      .customGrouping("word", new MyCustomStreamGrouping());

  Config conf = new Config();

  conf.setNumWorkers(2);

  StormSubmitter.submitTopology(args[0], conf, builder.createTopology());
}