Java Code Examples for backtype.storm.Config#setMaxTaskParallelism()

public static void main(String[] args) throws Exception {
    if (getProperties() == null || getProperties().isEmpty()) {
        System.out.println("Property file <ignite-storm.property> is not found or empty");
        return;
    }
    // Ignite Stream Ibolt
    final StormStreamer<String, String> stormStreamer = new StormStreamer<>();

    stormStreamer.setAutoFlushFrequency(10L);
    stormStreamer.setAllowOverwrite(true);
    stormStreamer.setCacheName(getProperties().getProperty("cache.name"));

    stormStreamer.setIgniteTupleField(getProperties().getProperty("tuple.name"));
    stormStreamer.setIgniteConfigFile(getProperties().getProperty("ignite.spring.xml"));


    TopologyBuilder builder = new TopologyBuilder();

    builder.setSpout("spout", new FileSourceSpout(), 1);
    builder.setBolt("limit", new SpeedLimitBolt(), 1).fieldsGrouping("spout", new Fields("trafficLog"));
    // set ignite bolt
    builder.setBolt("ignite-bolt", stormStreamer, STORM_EXECUTORS).shuffleGrouping("limit");

    Config conf = new Config();
    conf.setDebug(false);

    conf.setMaxTaskParallelism(1);
    LocalCluster cluster = new LocalCluster();
    cluster.submitTopology("speed-violation", conf, builder.createTopology());
    Thread.sleep(10000);
    cluster.shutdown();

}
 

/**
 * @param args
 * http://www.programcreek.com/java-api-examples/index.php?api=storm.kafka.KafkaSpout
 */
public static void main(String[] args) {
	try{
		//设置喷发节点并分配并发数，该并发数将会控制该对象在集群中的线程数（6个）
		String zkhost = "wxb-1:2181,wxb-2:2181,wxb-3:2181";
		String topic = "order";
		String groupId = "id";
		int spoutNum = 3;
		int boltNum = 1;
		ZkHosts zkHosts = new ZkHosts(zkhost);//kafaka所在的zookeeper
		SpoutConfig spoutConfig = new SpoutConfig(zkHosts, topic, "/order", groupId);  // create /order /id
		spoutConfig.scheme = new SchemeAsMultiScheme(new StringScheme());
		KafkaSpout kafkaSpout = new KafkaSpout(spoutConfig);

        TopologyBuilder builder = new TopologyBuilder();
        builder.setSpout("spout", kafkaSpout, spoutNum);
		builder.setBolt("check", new CheckOrderBolt(), boltNum).shuffleGrouping("spout");
        builder.setBolt("counter", new CounterBolt(),boltNum).shuffleGrouping("check");

        Config config = new Config();
        config.setDebug(true);
        
        if(args!=null && args.length > 0) {
            config.setNumWorkers(2);
            StormSubmitter.submitTopology(args[0], config, builder.createTopology());
        } else {        
            config.setMaxTaskParallelism(2);

            LocalCluster cluster = new LocalCluster();
            cluster.submitTopology("Wordcount-Topology", config, builder.createTopology());

            Thread.sleep(500000);

            cluster.shutdown();
        }
	}catch (Exception e) {
		e.printStackTrace();
	}
}
 

/**
 * The main method.
 * 
 * @param args
 *          the arguments
 */
public static void main(String[] args) {

  List<String> tmpArgs = new ArrayList<String>(Arrays.asList(args));

  int numWorker = StormSamoaUtils.numWorkers(tmpArgs);

  args = tmpArgs.toArray(new String[0]);

  // convert the arguments into Storm topology
  StormTopology stormTopo = StormSamoaUtils.argsToTopology(args);
  String topologyName = stormTopo.getTopologyName();

  Config conf = new Config();
  // conf.putAll(Utils.readStormConfig());
  conf.setDebug(false);

  // local mode
  conf.setMaxTaskParallelism(numWorker);

  backtype.storm.LocalCluster cluster = new backtype.storm.LocalCluster();
  cluster.submitTopology(topologyName, conf, stormTopo.getStormBuilder().createTopology());

  // Read local mode execution duration from property file
  Configuration stormConfig = StormSamoaUtils.getPropertyConfig(LocalStormDoTask.SAMOA_STORM_PROPERTY_FILE_LOC);
  long executionDuration= stormConfig.getLong(LocalStormDoTask.EXECUTION_DURATION_KEY);
  backtype.storm.utils.Utils.sleep(executionDuration * 1000);

  cluster.killTopology(topologyName);
  cluster.shutdown();

}
 

@Override
public Map<String, Object> getComponentConfiguration() {
    Config ret = new Config();
    ret.setMaxTaskParallelism(1);
    ret.registerSerialization(TransactionAttempt.class);
    return ret;
}
 

public static void main(String[] args) throws Exception {
	ParameterTool pt = ParameterTool.fromArgs(args);

	int par = pt.getInt("para");


	TridentTopology topology = new TridentTopology();
	Stream sourceStream = topology.newStream("source", new Generator(pt)).parallelismHint(pt.getInt("sourceParallelism"));

	Stream repart = sourceStream.partitionBy(new Fields("id"));
	for(int i = 0; i < pt.getInt("repartitions", 1) - 1; i++) {
		repart = repart.each(new Fields("id"), new IdentityEach(), new Fields("id"+i)).partitionBy(new Fields("id"+i));
	}
	repart.each(new Fields("id", "host", "time", "payload"), new Sink(pt), new Fields("dontcare")).parallelismHint(pt.getInt("sinkParallelism"));

	Config conf = new Config();
	conf.setDebug(false);

	if (!pt.has("local")) {
		conf.setNumWorkers(par);

		StormSubmitter.submitTopologyWithProgressBar("throughput-"+pt.get("name", "no_name"), conf, topology.build());
	}
	else {
		conf.setMaxTaskParallelism(par);

		LocalCluster cluster = new LocalCluster();
		cluster.submitTopology("throughput", conf, topology.build());

		Thread.sleep(30000);

		cluster.shutdown();
	}

}
 

public static void main(String[] args) throws AlreadyAliveException, InvalidTopologyException, UnknownHostException, InterruptedException {
	final ParameterTool pt = ParameterTool.fromArgs(args);

	TopologyBuilder builder = new TopologyBuilder();
	BrokerHosts hosts = new ZkHosts(pt.getRequired("zookeeper"));
	SpoutConfig spoutConfig = new SpoutConfig(hosts, pt.getRequired("topic"), "/" + pt.getRequired("topic"), UUID.randomUUID().toString());
	spoutConfig.scheme = new SchemeAsMultiScheme(new StringScheme());
	KafkaSpout kafkaSpout = new KafkaSpout(spoutConfig);
	builder.setSpout("source", kafkaSpout, pt.getInt("sourceParallelism"));

	builder.setBolt("sink", new Throughput.Sink(pt), pt.getInt("sinkParallelism")).noneGrouping("source");

	Config conf = new Config();
	conf.setDebug(false);

	if (!pt.has("local")) {
		conf.setNumWorkers(pt.getInt("par", 2));

		StormSubmitter.submitTopologyWithProgressBar("kafka-spout-"+pt.get("name", "no_name"), conf, builder.createTopology());
	} else {
		conf.setMaxTaskParallelism(pt.getInt("par", 2));

		LocalCluster cluster = new LocalCluster();
		cluster.submitTopology("kafka-spout", conf, builder.createTopology());

		Thread.sleep(300000);

		cluster.shutdown();
	}
}
 

public static void main(String[] args) throws Exception {
    // Ignite Stream Ibolt
    final StormStreamer<String, String> stormStreamer = new StormStreamer<>();

    stormStreamer.setAutoFlushFrequency(10L);
    stormStreamer.setAllowOverwrite(true);
    stormStreamer.setCacheName("testCache");
    stormStreamer.setIgniteTupleField("ignite");
    stormStreamer.setIgniteConfigFile("/Users/shamim/Development/workshop/assembla/ignite-book/chapters/chapter-cep-storm/src/main/resources/example-ignite.xml");

    //Used to build the topology
    TopologyBuilder builder = new TopologyBuilder();
    //Add the spout, with a name of 'spout'
    //and parallelism hint of 5 executors
    builder.setSpout("spout", new RandomSentenceSpout(), 5);
    //Add the SplitSentence bolt, with a name of 'split'
    //and parallelism hint of 8 executors
    //shufflegrouping subscribes to the spout, and equally distributes
    //tuples (sentences) across instances of the SplitSentence bolt
    builder.setBolt("split", new SplitSentence(), 8).shuffleGrouping("spout");
    //Add the counter, with a name of 'count'
    //and parallelism hint of 12 executors
    //fieldsgrouping subscribes to the split bolt, and
    //ensures that the same word is sent to the same instance (group by field 'word')
    builder.setBolt("count", new WordCount(), 12).fieldsGrouping("split", new Fields("word"));
    // set ignite bolt
    builder.setBolt("ignite-bolt", stormStreamer,STORM_EXECUTORS).shuffleGrouping("count");

    //new configuration
    Config conf = new Config();
    //Set to false to disable debug information
    // when running in production mode.
    conf.setDebug(false);

    //If there are arguments, we are running on a cluster
    if (args != null && args.length > 0) {
        //parallelism hint to set the number of workers
        conf.setNumWorkers(3);
        //submit the topology
        StormSubmitter.submitTopology(args[0], conf, builder.createTopology());
    }
    //Otherwise, we are running locally
    else {
        //Cap the maximum number of executors that can be spawned
        //for a component to 3
        conf.setMaxTaskParallelism(3);
        //LocalCluster is used to run locally
        LocalCluster cluster = new LocalCluster();
        //submit the topology
        cluster.submitTopology("word-count", conf, builder.createTopology());
        //sleep
        Thread.sleep(10000);
        //shut down the cluster
        cluster.shutdown();
    }
}
 

@Override
public Map<String, Object> getComponentConfiguration() {
    Config ret = new Config();
    ret.setMaxTaskParallelism(1);
    return ret;
}
 

@Override
public Map getComponentConfiguration() {
    Config conf = new Config();
    conf.setMaxTaskParallelism(1);
    return conf;
}
 

@Override
public Map<String, Object> getComponentConfiguration() {
    Config conf = new Config();
    conf.setMaxTaskParallelism(1);
    return conf;
}
 

@Override
public Map<String, Object> getComponentConfiguration() {
    Config ret = new Config();
    ret.setMaxTaskParallelism(1);
    return ret;
}
 

@Override
public Map getComponentConfiguration() {
    Config conf = new Config();
    conf.setMaxTaskParallelism(1);
    return conf;
}
 

public static final void main(final String[] args) throws Exception {
	final ApplicationContext applicationContext = new ClassPathXmlApplicationContext("applicationContext.xml");

	final JmsProvider jmsProvider = new SpringJmsProvider(applicationContext, "jmsConnectionFactory",
			                                                          "notificationQueue");

	final TopologyBuilder topologyBuilder = new TopologyBuilder();

	final JmsBolt jmsBolt = new JmsBolt();
	jmsBolt.setJmsProvider(jmsProvider);
	jmsBolt.setJmsMessageProducer((session, input) -> {
           final String json = "{\"stateCode\":\"" + input.getString(0) + "\", \"sentiment\":" + input.getInteger(1) + "}";
           return session.createTextMessage(json);
       });

	try {
		final Config config = new Config();
		config.setMessageTimeoutSecs(120);
		config.setDebug(true);

		topologyBuilder.setSpout("twitterspout", new TwitterSpout());
		topologyBuilder.setBolt("statelocatorbolt", new StateLocatorBolt())
				.shuffleGrouping("twitterspout");
		topologyBuilder.setBolt("sentimentcalculatorbolt", new SentimentCalculatorBolt())
				.fieldsGrouping("statelocatorbolt", new Fields("state"));
		topologyBuilder.setBolt("jmsBolt", jmsBolt).fieldsGrouping("sentimentcalculatorbolt", new Fields("stateCode"));

		//Submit it to the cluster, or submit it locally
		if (null != args && 0 < args.length) {
			config.setNumWorkers(3);
			StormSubmitter.submitTopology(args[0], config, topologyBuilder.createTopology());
		} else {
			config.setMaxTaskParallelism(10);
			final LocalCluster localCluster = new LocalCluster();
			localCluster.submitTopology(Constants.TOPOLOGY_NAME, config, topologyBuilder.createTopology());
			//Run this topology for 600 seconds so that we can complete processing of decent # of tweets.
			Utils.sleep(600 * 1000);

			LOGGER.info("Shutting down the cluster...");
			localCluster.killTopology(Constants.TOPOLOGY_NAME);
			localCluster.shutdown();

			Runtime.getRuntime().addShutdownHook(new Thread()	{
				@Override
				public void run()	{
					LOGGER.info("Shutting down the cluster...");
					localCluster.killTopology(Constants.TOPOLOGY_NAME);
					localCluster.shutdown();
				}
			});
		}
	} catch (final Exception exception) {
		//Deliberate no op;
		exception.printStackTrace();
	}
	LOGGER.info("\n\n\n\t\t*****Please clean your temp folder \"{}\" now!!!*****", System.getProperty("java.io.tmpdir"));
}
 

private static void loadTopologyPropertiesAndSubmit(Properties properties,
		Config config) throws Exception {
	
	String stormExecutionMode = properties.getProperty("storm.execution.mode","local");
	int stormWorkersNumber = Integer.parseInt(properties.getProperty("storm.workers.number","2"));
	int maxTaskParallism = Integer.parseInt(properties.getProperty("storm.max.task.parallelism","2"));
	String topologyName = properties.getProperty("storm.topology.name","topologyName");
	String zookeeperHosts = properties.getProperty("zookeeper.hosts");
	int topologyBatchEmitMillis = Integer.parseInt(
			properties.getProperty("storm.topology.batch.interval.miliseconds","2000"));
	
	// How often a batch can be emitted in a Trident topology.
	config.put(Config.TOPOLOGY_TRIDENT_BATCH_EMIT_INTERVAL_MILLIS, topologyBatchEmitMillis);
	config.setNumWorkers(stormWorkersNumber);
	config.setMaxTaskParallelism(maxTaskParallism);
	
	AuditActiveLoginsTopology auditActiveLoginsTopology = new AuditActiveLoginsTopology(zookeeperHosts);
	StormTopology stormTopology = auditActiveLoginsTopology.buildTopology(properties);
	
	// Elastic Search specific properties
	config.put(StormElasticSearchConstants.ES_HOST, properties.getProperty("elasticsearch.host", "localhost"));
	config.put(StormElasticSearchConstants.ES_PORT, (Integer.parseInt(properties.getProperty("elasticsearch.port", "9300"))));
	config.put(StormElasticSearchConstants.ES_CLUSTER_NAME, properties.getProperty("elasticsearch.cluster.name"));
	config.put("elasticsearch.index", properties.getProperty("elasticsearch.index"));
	config.put("elasticsearch.type", properties.getProperty("elasticsearch.type"));

	switch (stormExecutionMode){
		case ("cluster"):
			String nimbusHost = properties.getProperty("storm.nimbus.host","localhost");
			String nimbusPort = properties.getProperty("storm.nimbus.port","6627");
			config.put(Config.NIMBUS_HOST, nimbusHost);
			config.put(Config.NIMBUS_THRIFT_PORT, Integer.parseInt(nimbusPort));
			config.put(Config.STORM_ZOOKEEPER_PORT, parseZkPort(zookeeperHosts));
			config.put(Config.STORM_ZOOKEEPER_SERVERS, parseZkHosts(zookeeperHosts));
			StormSubmitter.submitTopology(topologyName, config, stormTopology);
			break;
		case ("local"):
		default:
			int localTimeExecution = Integer.parseInt(properties.getProperty("storm.local.execution.time","20000"));
			LocalCluster cluster = new LocalCluster();
			cluster.submitTopology(topologyName, config, stormTopology);
			Thread.sleep(localTimeExecution);
			cluster.killTopology(topologyName);
			cluster.shutdown();
			System.exit(0);
	}		
}
 

/**
 * The main method.
 * 
 * @param args
 *          the arguments
 */
public static void main(String[] args) {

  List<String> tmpArgs = new ArrayList<String>(Arrays.asList(args));

  boolean isLocal = isLocal(tmpArgs);
  int numWorker = StormSamoaUtils.numWorkers(tmpArgs);

  args = tmpArgs.toArray(new String[0]);

  // convert the arguments into Storm topology
  StormTopology stormTopo = StormSamoaUtils.argsToTopology(args);
  String topologyName = stormTopo.getTopologyName();

  Config conf = new Config();
  conf.putAll(Utils.readStormConfig());
  conf.setDebug(false);

  if (isLocal) {
    // local mode
    conf.setMaxTaskParallelism(numWorker);

    backtype.storm.LocalCluster cluster = new backtype.storm.LocalCluster();
    cluster.submitTopology(topologyName, conf, stormTopo.getStormBuilder().createTopology());

    backtype.storm.utils.Utils.sleep(600 * 1000);

    cluster.killTopology(topologyName);
    cluster.shutdown();

  } else {
    // cluster mode
    conf.setNumWorkers(numWorker);
    try {
      backtype.storm.StormSubmitter.submitTopology(topologyName, conf,
          stormTopo.getStormBuilder().createTopology());
    } catch (backtype.storm.generated.AlreadyAliveException ale) {
      ale.printStackTrace();
    } catch (backtype.storm.generated.InvalidTopologyException ite) {
      ite.printStackTrace();
    }
  }
}
 

@Override
public Map<String, Object> getComponentConfiguration() {
  Config ret = new Config();
  ret.setMaxTaskParallelism(1);
  return ret;
}
 

/**
 * The main method.
 * 
 * @param args the arguments
 */
public static void main(String[] args) {

	 List<String> tmpArgs = new ArrayList<String>(Arrays.asList(args));
	
	boolean isLocal = isLocal(tmpArgs);
	int numWorker = StormSamoaUtils.numWorkers(tmpArgs);
	
	args = tmpArgs.toArray(new String[0]);
	
	//convert the arguments into Storm topology
	StormTopology stormTopo = StormSamoaUtils.argsToTopology(args);
	String topologyName = stormTopo.getTopologyName();
	
   	Config conf = new Config();
   	conf.putAll(Utils.readStormConfig());
   	conf.setDebug(false);
   			
   	
	if(isLocal){
		//local mode
		conf.setMaxTaskParallelism(numWorker);
		
		backtype.storm.LocalCluster cluster = new backtype.storm.LocalCluster();
		cluster.submitTopology(topologyName , conf, stormTopo.getStormBuilder().createTopology());
		
		backtype.storm.utils.Utils.sleep(600*1000);
		
		cluster.killTopology(topologyName);
		cluster.shutdown();
		
	}else{
		//cluster mode
		conf.setNumWorkers(numWorker);
		try {
			backtype.storm.StormSubmitter.submitTopology(topologyName, conf,
					stormTopo.getStormBuilder().createTopology());
		} catch (backtype.storm.generated.AlreadyAliveException ale) {
			ale.printStackTrace();
		} catch (backtype.storm.generated.InvalidTopologyException ite) {
			ite.printStackTrace();
		}
	}
}
 

public static void main(String[] args) throws Exception {
	ParameterTool pt = ParameterTool.fromArgs(args);

	int par = pt.getInt("para");

	TopologyBuilder builder = new TopologyBuilder();

	builder.setSpout("source0", new Generator(pt), pt.getInt("sourceParallelism"));

	//builder.setBolt("sink", new Sink(pt), pt.getInt("sinkParallelism")).noneGrouping("source0");
	builder.setBolt("sink", new Sink(pt), pt.getInt("sinkParallelism")).localOrShuffleGrouping("source0");


	Config conf = new Config();
	conf.setDebug(false);

	conf.setMaxSpoutPending(pt.getInt("maxPending", 1000));
	//System.exit(1);

	if (!pt.has("local")) {
		conf.setNumWorkers(par);

		StormSubmitter.submitTopologyWithProgressBar("forward-throughput-"+pt.get("name", "no_name"), conf, builder.createTopology());
	}
	else {
		conf.setMaxTaskParallelism(par);

		LocalCluster cluster = new LocalCluster();
		cluster.submitTopology("forward-throughput", conf, builder.createTopology());

		Thread.sleep(300000);

		cluster.shutdown();
	}

}
 

public static void main(String[] args) throws Exception {
	ParameterTool pt = ParameterTool.fromArgs(args);

	int par = pt.getInt("para");

	TridentTopology topology = new TridentTopology();
	Stream sourceStream = topology.newStream("source", new Generator(pt)).parallelismHint(pt.getInt("sourceParallelism"));
	sourceStream.localOrShuffle().each(FIELDS, new Sink(pt), new Fields("dontcare"));

	Config conf = new Config();
	conf.setDebug(false);

//	conf.setMaxSpoutPending(pt.getInt("maxPending", 1000));

	//System.exit(1);

	if (!pt.has("local")) {
		conf.setNumWorkers(par);

		StormSubmitter.submitTopologyWithProgressBar("forward-throughput-"+pt.get("name", "no_name"), conf, topology.build());
	}
	else {
		conf.setMaxTaskParallelism(par);

		LocalCluster cluster = new LocalCluster();
		cluster.submitTopology("forward-throughput", conf, topology.build());

		Thread.sleep(300000);

		cluster.shutdown();
	}

}
 

public static void main(String[] args) throws Exception {

    TopologyBuilder builder = new TopologyBuilder();

    builder.setSpout("spout", new RandomSentenceSpout(), 5);

    builder.setBolt("split", new SplitSentence(), 8).shuffleGrouping("spout");
    builder.setBolt("count", new WordCount(), 12).fieldsGrouping("split", new Fields("word"));

    Config conf = new Config();
    conf.setDebug(true);


    if (args != null && args.length > 0) {
      conf.setNumWorkers(3);

      StormSubmitter.submitTopologyWithProgressBar(args[0], conf, builder.createTopology());
    }
    else {
      conf.setMaxTaskParallelism(3);

      LocalCluster cluster = new LocalCluster();
      cluster.submitTopology("word-count", conf, builder.createTopology());

      Thread.sleep(10000);

      cluster.shutdown();
    }
  }