Java Code Examples for org.apache.storm.Config#put()

public static void main(String[] args) throws Exception {
  if (args.length != 1) {
    throw new RuntimeException("Specify topology name");
  }
  TopologyBuilder builder = new TopologyBuilder();

  int spouts = 2;
  int bolts = 2;
  builder.setSpout("word", new AckingTestWordSpout(), spouts);
  builder.setBolt("exclaim1", new ExclamationBolt(), bolts)
      .shuffleGrouping("word");

  Config conf = new Config();
  conf.setDebug(true);

  // Put an arbitrary large number here if you don't want to slow the topology down
  conf.setMaxSpoutPending(1000 * 1000 * 1000);

  // To enable acking, we need to setEnableAcking true
  conf.setNumAckers(1);
  conf.put(Config.TOPOLOGY_WORKER_CHILDOPTS, "-XX:+HeapDumpOnOutOfMemoryError");

  // Set the number of workers or stream managers
  conf.setNumWorkers(2);
  StormSubmitter.submitTopology(args[0], conf, builder.createTopology());
}
 

private void runTopology(File responderFile) throws Exception
{
  MkClusterParam mkClusterParam = new MkClusterParam();
  // The test sometimes fails because of timing issues when more than 1 supervisor set.
  mkClusterParam.setSupervisors(1);

  // Maybe using "withSimulatedTimeLocalCluster" would be better to avoid worrying about timing.
  Config conf = PirkTopology.createStormConf();
  conf.put(StormConstants.OUTPUT_FILE_KEY, responderFile.getAbsolutePath());
  conf.put(StormConstants.N_SQUARED_KEY, nSquared.toString());
  conf.put(StormConstants.QUERY_INFO_KEY, queryInfo.toMap());
  // conf.setDebug(true);
  mkClusterParam.setDaemonConf(conf);

  TestJob testJob = createPirkTestJob(conf);
  Testing.withLocalCluster(mkClusterParam, testJob);
  // Testing.withSimulatedTimeLocalCluster(mkClusterParam, testJob);
}
 

public static void submitJob(String principal, String keytab, String esNodes) throws Exception {
    List doc1 = Collections.singletonList("{\"reason\" : \"business\",\"airport\" : \"SFO\"}");
    List doc2 = Collections.singletonList("{\"participants\" : 5,\"airport\" : \"OTP\"}");

    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("Input", new TestSpout(ImmutableList.of(doc1, doc2), new Fields("json"), true));
    builder.setBolt("ES", new EsBolt("storm-test"))
            .shuffleGrouping("Input")
            .addConfiguration(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS, 2);

    // Nimbus needs to be started with the cred renewer and credentials plugins set in its config file

    Config conf = new Config();
    List<Object> plugins = new ArrayList<Object>();
    plugins.add(AutoElasticsearch.class.getName());
    conf.put(Config.TOPOLOGY_AUTO_CREDENTIALS, plugins);
    conf.put(ConfigurationOptions.ES_NODES, esNodes);
    conf.put(ConfigurationOptions.ES_SECURITY_AUTHENTICATION, "kerberos");
    conf.put(ConfigurationOptions.ES_NET_SPNEGO_AUTH_ELASTICSEARCH_PRINCIPAL, "HTTP/[email protected]");
    conf.put(ConfigurationOptions.ES_INPUT_JSON, "true");
    StormSubmitter.submitTopology("test-run", conf, builder.createTopology());
}
 

private void start(StormTopology topology, boolean runAsLocal) throws Exception {
    Config conf = new Config();
    conf.put(FullPullConstants.FULL_SPLITTER_TOPOLOGY_ID, fullSplitterTopologyId);
    conf.put(FullPullConstants.FULL_PULLER_TOPOLOGY_ID, fullPullerTopologyId);
    conf.put(FullPullConstants.DS_NAME, topologyId);
    conf.put(FullPullConstants.ZKCONNECT, zkConnect);
    conf.put(Config.TOPOLOGY_WORKER_CHILDOPTS, commonConfSplit.getProperty(FullPullConstants.TOPOLOGY_WORKER_CHILDOPTS));
    //设置message超时时间为，保证每个分片都能在该内拉完数据
    conf.setMessageTimeoutSecs(Integer.parseInt(commonConfSplit.getProperty(FullPullConstants.STORM_MESSAGE_TIMEOUT)));
    conf.setMaxSpoutPending(Integer.parseInt(commonConfSplit.getProperty(FullPullConstants.STORM_MAX_SPOUT_PENDING)));
    conf.setNumWorkers(Integer.parseInt(commonConfSplit.getProperty(FullPullConstants.STORM_NUM_WORKERS)));
    conf.setDebug(true);

    if (runAsLocal) {
        conf.setMaxTaskParallelism(3);
        LocalCluster cluster = new LocalCluster();
        cluster.submitTopology(topologyName, conf, topology);
    } else {
        StormSubmitter.submitTopology(topologyName, conf, topology);
    }
}
 

private void start(StormTopology topology, boolean runAsLocal) throws Exception {
    Config conf = new Config();
    conf.put(com.creditease.dbus.commons.Constants.ZOOKEEPER_SERVERS, zkConnect);
    conf.put(Constants.TOPOLOGY_ID, topologyId);
    conf.setMessageTimeoutSecs(Integer.parseInt(properties.getProperty(Constants.LOG_MESSAGE_TIMEOUT)));
    //conf.setMaxSpoutPending(30);
    conf.setDebug(true);
    conf.setNumWorkers(Integer.parseInt(properties.getProperty(Constants.LOG_NUMWORKERS)));

    if (runAsLocal) {
        conf.setMaxTaskParallelism(10);
        LocalCluster cluster = new LocalCluster();
        cluster.submitTopology(topologyName, conf, topology);
    } else {
        StormSubmitter.submitTopology(topologyName, conf, topology);
    }
}
 

@Test
public void testExclusion() throws IOException {
    Config conf = new Config();
    conf.put(TextExtractor.EXCLUDE_PARAM_NAME, "STYLE");

    TextExtractor extractor = new TextExtractor(conf);

    String content = "<html>the<style>main</style>content of the page</html>";

    Document jsoupDoc = Parser.htmlParser().parseInput(content,
            "http://stormcrawler.net");
    String text = extractor.text(jsoupDoc.body());

    assertEquals("the content of the page", text);
}
 

@Test
public void testMainContent() throws IOException {
    Config conf = new Config();
    conf.put(TextExtractor.INCLUDE_PARAM_NAME, "DIV[id=\"maincontent\"]");

    TextExtractor extractor = new TextExtractor(conf);

    String content = "<html>the<div id='maincontent'>main<div>content</div></div>of the page</html>";

    Document jsoupDoc = Parser.htmlParser().parseInput(content,
            "http://stormcrawler.net");
    String text = extractor.text(jsoupDoc.body());

    assertEquals("main content", text);
}
 

public static Config createStormConf()
{

  Boolean limitHitsPerSelector = Boolean.parseBoolean(SystemConfiguration.getProperty("pir.limitHitsPerSelector"));
  Integer maxHitsPerSelector = Integer.parseInt(SystemConfiguration.getProperty("pir.maxHitsPerSelector"));
  Integer rowDivisions = Integer.parseInt(SystemConfiguration.getProperty("storm.rowDivs", "1"));

  Config conf = new Config();
  conf.setNumAckers(Integer.parseInt(SystemConfiguration.getProperty("storm.numAckers", numWorkers.toString())));
  conf.setMaxSpoutPending(Integer.parseInt(SystemConfiguration.getProperty("storm.maxSpoutPending", "300")));
  conf.setNumWorkers(numWorkers);
  conf.setDebug(false);
  // conf.setNumEventLoggers(2);

  conf.put(Config.TOPOLOGY_EXECUTOR_RECEIVE_BUFFER_SIZE, SystemConfiguration.getIntProperty("storm.executor.receiveBufferSize", 1024));
  conf.put(Config.TOPOLOGY_EXECUTOR_SEND_BUFFER_SIZE, SystemConfiguration.getIntProperty("storm.executor.sendBufferSize", 1024));
  conf.put(Config.TOPOLOGY_TRANSFER_BUFFER_SIZE, SystemConfiguration.getIntProperty("storm.transferBufferSize", 32));
  conf.put(Config.WORKER_HEAP_MEMORY_MB, SystemConfiguration.getIntProperty("storm.worker.heapMemory", 750));
  conf.put(Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB, Double.parseDouble(SystemConfiguration.getProperty("storm.componentOnheapMem", "128")));

  // Pirk parameters to send to bolts
  conf.put(StormConstants.ALLOW_ADHOC_QSCHEMAS_KEY, SystemConfiguration.getProperty("pir.allowAdHocQuerySchemas", "false").equals("true"));
  conf.put(StormConstants.QSCHEMA_KEY, SystemConfiguration.getProperty("query.schemas"));
  conf.put(StormConstants.DSCHEMA_KEY, SystemConfiguration.getProperty("data.schemas"));
  conf.put(StormConstants.HDFS_URI_KEY, hdfsUri);
  conf.put(StormConstants.QUERY_FILE_KEY, queryFile);
  conf.put(StormConstants.USE_HDFS, useHdfs);
  conf.put(StormConstants.OUTPUT_FILE_KEY, outputPath);
  conf.put(StormConstants.LIMIT_HITS_PER_SEL_KEY, limitHitsPerSelector);
  conf.put(StormConstants.MAX_HITS_PER_SEL_KEY, maxHitsPerSelector);
  conf.put(StormConstants.SPLIT_PARTITIONS_KEY, splitPartitions);
  conf.put(StormConstants.SALT_COLUMNS_KEY, saltColumns);
  conf.put(StormConstants.ROW_DIVISIONS_KEY, rowDivisions);
  conf.put(StormConstants.ENCROWCALCBOLT_PARALLELISM_KEY, encrowcalcboltParallelism);
  conf.put(StormConstants.ENCCOLMULTBOLT_PARALLELISM_KEY, enccolmultboltParallelism);

  return conf;
}
 

public static Config submitTopology(ILocalCluster stormCluster, String topologyName,
                                    StormTopology stormTopology) throws Exception {
    Config stormConf = new Config();
    stormConf.putAll(Utils.readDefaultConfig());
    stormConf.put("storm.cluster.mode", "local");
    stormConf.setDebug(true);
    stormConf.setMaxTaskParallelism(3);
    stormConf.put(Config.STORM_TOPOLOGY_SUBMISSION_NOTIFIER_PLUGIN,
            org.apache.atlas.storm.hook.StormAtlasHook.class.getName());

    stormCluster.submitTopology(topologyName, stormConf, stormTopology);

    Thread.sleep(10000);
    return stormConf;
}
 

@Override
public Map<String, Object> getComponentConfiguration() {
    // We setup tick tuples to expire tuples when one of the streams has time based retention
    // to ensure expiration occurs even if there is no incoming data
    Config conf = new Config();
    if (needTicks())
        conf.put(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS, 1);
    return conf;
}
 

private void start(StormTopology topology, boolean runAsLocal) throws Exception {
    Config conf = new Config();
    conf.put(com.creditease.dbus.commons.Constants.ZOOKEEPER_SERVERS, zkConnect);
    conf.put(Constants.TOPOLOGY_ID, topologyId);
    conf.put(Constants.TOPOLOGY_ALIAS, alias);
    conf.put(Constants.ROUTER_PROJECT_NAME, projectName);

    String workerChildOpts = routerConf.getProperty(DBusRouterConstants.STORM_TOPOLOGY_WORKER_CHILDOPTS, "-Xmx2g");
    conf.put(Config.TOPOLOGY_WORKER_CHILDOPTS, workerChildOpts);

    int msgTimeout = Integer.valueOf(routerConf.getProperty(DBusRouterConstants.STORM_MESSAGE_TIMEOUT, "10"));
    conf.setMessageTimeoutSecs(msgTimeout);

    int maxSpoutPending = Integer.valueOf(routerConf.getProperty(DBusRouterConstants.STORM_MAX_SPOUT_PENDING, "100"));
    conf.setMaxSpoutPending(maxSpoutPending);
    conf.setDebug(true);

    int numWorks = Integer.valueOf(routerConf.getProperty(DBusRouterConstants.STORM_NUM_WORKS, "1"));
    conf.setNumWorkers(numWorks);

    if (runAsLocal) {
        conf.setMaxTaskParallelism(10);
        LocalCluster cluster = new LocalCluster();
        cluster.submitTopology(topologyName, conf, topology);
    } else {
        StormSubmitter.submitTopology(topologyName, conf, topology);
    }
}
 

private static void copyPropertiesIntoCfg(Config cfg) {
    Properties props = TestSettings.TESTING_PROPS;

    for (String property : props.stringPropertyNames()) {
        cfg.put(property, props.get(property));
    }
}
 

public void buildTopology(String[] args) {
    //TODO
    if (parseCommandArgs(args) != 0) {
        return;
    }
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("CanalClientSpout", new CanalClientSpout(), 1);
    builder.setBolt("KafkaProducerBolt", new KafkaProducerBolt(), 1).shuffleGrouping("CanalClientSpout");

    Config conf = new Config();
    conf.put(Constants.ZOOKEEPER_SERVERS, zkServers);
    conf.put(Constants.EXTRACTOR_TOPOLOGY_ID, extractorTopologyId);
    logger.info(Constants.ZOOKEEPER_SERVERS + "=" + zkServers);
    logger.info(Constants.EXTRACTOR_TOPOLOGY_ID + "=" + extractorTopologyId);
    conf.setNumWorkers(1);
    conf.setMaxSpoutPending(50);
    conf.setMessageTimeoutSecs(120);
    if (!runAsLocal) {
        conf.setDebug(false);
        try {
            //StormSubmitter.submitTopology("extractorTopologyId", conf, builder.createTopology());
            StormSubmitter.submitTopology(extractorTopologyId, conf, builder.createTopology());
        } catch (Exception e) {
            e.printStackTrace();
        }
    } else {
        conf.setDebug(false);
        LocalCluster cluster = new LocalCluster();
        //cluster.submitTopology("extractorTopologyId", conf, builder.createTopology());
        cluster.submitTopology(extractorTopologyId, conf, builder.createTopology());
    }
}
 

public static Config submitTopology(ILocalCluster stormCluster, String topologyName,
                                    StormTopology stormTopology) throws Exception {
    Config stormConf = new Config();
    stormConf.putAll(Utils.readDefaultConfig());
    stormConf.put("storm.cluster.mode", "local");
    stormConf.setDebug(true);
    stormConf.setMaxTaskParallelism(3);
    stormConf.put(Config.STORM_TOPOLOGY_SUBMISSION_NOTIFIER_PLUGIN,
            org.apache.atlas.storm.hook.StormAtlasHook.class.getName());

    stormCluster.submitTopology(topologyName, stormConf, stormTopology);

    Thread.sleep(10000);
    return stormConf;
}
 

public static void main(String[] args) throws InterruptedException, AlreadyAliveException, InvalidTopologyException {
 	//定义拓扑
     TopologyBuilder builder = new TopologyBuilder();
     builder.setSpout("word-reader" , new WordReader());
     builder.setBolt("word-normalizer" , new WordNormalizer()).shuffleGrouping("word-reader" );
     builder.setBolt("word-counter" , new WordCounter()).fieldsGrouping("word-normalizer" , new Fields("word"));
     StormTopology topology = builder.createTopology();
     //配置
     
     Config conf = new Config();
     String fileName ="words.txt" ;
     conf.put("fileName" , fileName );
     conf.setDebug(false);
 
      //运行拓扑
      System.out.println("开始...");
      if(args !=null&&args.length>0){ //有参数时，表示向集群提交作业，并把第一个参数当做topology名称
     	 System.out.println("远程模式");
          try {
	StormSubmitter.submitTopology(args[0], conf, topology);
} catch (AuthorizationException e) {
	e.printStackTrace();
}
    } else{//没有参数时，本地提交
      //启动本地模式
 	 System.out.println("本地模式");
      LocalCluster cluster = new LocalCluster();
      cluster.submitTopology("Getting-Started-Topologie" , conf , topology );
      Thread.sleep(5000);
      //关闭本地集群
      cluster.shutdown();
    }
      System.out.println("结束");
    
 }
 

public static void main(String[] args)  {
		//定义一个拓扑
		TopologyBuilder builder=new TopologyBuilder();
		//设置两个Executeor(线程)，默认一个
		builder.setSpout(test_spout, new TestSpout(),2);
		//shuffleGrouping:表示是随机分组
		//设置两个Executeor(线程)，和两个task
		builder.setBolt(test_bolt, new TestBolt(),2).setNumTasks(2).shuffleGrouping(test_spout);
		//fieldsGrouping:表示是按字段分组
		//设置两个Executeor(线程)，和两个task
		builder.setBolt(test2_bolt, new Test2Bolt(),2).setNumTasks(2).fieldsGrouping(test_bolt, new Fields("count"));
		Config conf = new Config();
		conf.put("test", "test");
		try{
		  //运行拓扑
	       if(args !=null&&args.length>0){ //有参数时，表示向集群提交作业，并把第一个参数当做topology名称
	       	 System.out.println("运行远程模式");
			 StormSubmitter.submitTopology(args[0], conf, builder.createTopology());
	      } else{//没有参数时，本地提交
	        //启动本地模式
	     	System.out.println("运行本地模式");
	        LocalCluster cluster = new LocalCluster();
	        cluster.submitTopology("Word-counts" ,conf,  builder.createTopology() );
	        Thread.sleep(20000);
//	        //关闭本地集群
	        cluster.shutdown();
	      }
		}catch (Exception e){
			e.printStackTrace();
		}
	}
 

public static void main(String[] args)  {
		// TODO Auto-generated method stub
		//定义一个拓扑
		TopologyBuilder builder=new TopologyBuilder();
		builder.setSpout(str1, new TestSpout());
		builder.setBolt(str2, new TestBolt()).shuffleGrouping(str1);
		Config conf = new Config();
		conf.put("test", "test");
		try{
		  //运行拓扑
	       if(args !=null&&args.length>0){ //有参数时，表示向集群提交作业，并把第一个参数当做topology名称
	       	 System.out.println("远程模式");
			 StormSubmitter.submitTopology(args[0], conf, builder.createTopology());
	      } else{//没有参数时，本地提交
	        //启动本地模式
	     	System.out.println("本地模式");
	        LocalCluster cluster = new LocalCluster();
	        cluster.submitTopology("111" ,conf,  builder.createTopology() );
//	        Thread.sleep(2000);
//	        //关闭本地集群
//	        cluster.shutdown();
	      }
		}catch (Exception e){
			e.printStackTrace();
		}
		
	}
 

private void start(StormTopology topology, boolean runAsLocal) throws Exception {

        Config conf = new Config();

        // 启动类型为all，或者dispatcher
        if (topologyType.equals(Constants.TopologyType.ALL) || topologyType.equals(Constants.TopologyType.DISPATCHER)) {
            /**
             * dispatcher配置
             */

            conf.put(com.creditease.dbus.commons.Constants.ZOOKEEPER_SERVERS, zookeeper);
            conf.put(com.creditease.dbus.commons.Constants.TOPOLOGY_ID, dispatcherTopologyId);
            logger.info(com.creditease.dbus.commons.Constants.ZOOKEEPER_SERVERS + "=" + zookeeper);
            logger.info(com.creditease.dbus.commons.Constants.TOPOLOGY_ID + "=" + dispatcherTopologyId);
        }

        // 启动类型为all，或者appender
        if (topologyType.equals(Constants.TopologyType.ALL) || topologyType.equals(Constants.TopologyType.APPENDER)) {

            /**
             * appender配置
             */
            conf.put(Constants.StormConfigKey.TOPOLOGY_ID, appenderTopologyId);
            conf.put(Constants.StormConfigKey.ZKCONNECT, zookeeper);
            conf.put(Constants.StormConfigKey.DATASOURCE, datasource);
        }

//        conf.put(Config.TOPOLOGY_TRANSFER_BUFFER_SIZE, 4096);
//        conf.put(Config.TOPOLOGY_EXECUTOR_RECEIVE_BUFFER_SIZE, 4096);
//        conf.put(Config.TOPOLOGY_EXECUTOR_SEND_BUFFER_SIZE, 4096);

        conf.setDebug(true);

        conf.setNumAckers(1);
        //设置worker数
        conf.setNumWorkers(1);
        //设置任务在发出后，但还没处理完成的中间状态任务的最大数量, 如果没有设置最大值为50
        int MaxSpoutPending = getConfigureValueWithDefault(Constants.ConfigureKey.MAX_SPOUT_PENDING, 50);
        conf.setMaxSpoutPending(MaxSpoutPending);
        //设置任务在多久之内没处理完成，则这个任务处理失败
        conf.setMessageTimeoutSecs(120);

        String opts = getWorkerChildopts();
        if (opts != null && opts.trim().length() > 0) {
            conf.put(Config.TOPOLOGY_WORKER_CHILDOPTS, opts);
        }

//        conf.put(Config.TOPOLOGY_SKIP_MISSING_KRYO_REGISTRATIONS, true);
//        conf.registerSerialization(org.apache.avro.util.Utf8.class);
//        conf.registerSerialization(com.creditease.dbus.commons.DBusConsumerRecord.class);
//        conf.registerSerialization(org.apache.kafka.common.record.TimestampType.class);
//        conf.registerSerialization(com.creditease.dbus.stream.common.appender.bean.EmitData.class);
//        conf.registerSerialization(com.creditease.dbus.stream.common.appender.enums.Command.class);
//        conf.registerSerialization(org.apache.avro.generic.GenericData.class);
//        conf.registerSerialization(com.creditease.dbus.stream.oracle.appender.avro.GenericData.class);
//        conf.registerSerialization(com.creditease.dbus.commons.DbusMessage12.class);
//        conf.registerSerialization(com.creditease.dbus.commons.DbusMessage12.Schema12.class);
//        conf.registerSerialization(com.creditease.dbus.commons.DbusMessage13.Schema13.class);
//        conf.registerSerialization(com.creditease.dbus.commons.DbusMessage13.class);
//        conf.registerSerialization(com.creditease.dbus.commons.DbusMessage.Field.class);
//        conf.registerSerialization(com.creditease.dbus.commons.DbusMessage.Payload.class);
//        conf.registerSerialization(com.creditease.dbus.commons.DbusMessage.Protocol.class);
//        conf.registerSerialization(com.creditease.dbus.commons.DbusMessage.ProtocolType.class);
//        conf.registerSerialization(com.creditease.dbus.stream.oracle.appender.bolt.processor.appender.OraWrapperData.class);
//        conf.registerSerialization(com.creditease.dbus.stream.common.appender.spout.cmds.TopicResumeCmd.class);

        if (runAsLocal) {
            LocalCluster cluster = new LocalCluster();
            cluster.submitTopology(topologyId, conf, topology);
            /*String cmd;
            do {
                cmd = System.console().readLine();
            } while (!cmd.equals("exit"));
            cluster.shutdown();*/
        } else {
            StormSubmitter.submitTopology(topologyId, conf, topology);
        }
    }
 

@Override
public Map<String, Object> getComponentConfiguration() {
    Config conf = new Config();
    conf.put(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS, freqStats);
    return conf;
}
 

@Override
public Map<String, Object> getComponentConfiguration() {
    Config conf = new Config();
    conf.put(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS, 1);
    return conf;
}