org.apache.storm.StormSubmitter Java Examples

The following examples show how to use org.apache.storm.StormSubmitter. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TopologyStarter.java    From breeze with Apache License 2.0 6 votes vote down vote up
@Override
public void run() {
	properties.put(Config.TOPOLOGY_NAME, ID);
	Config config = stormConfig(properties);
	ApplicationContext spring = SingletonApplicationContext.loadXml(config, MAIN_CONTEXT);
	try {
		StormTopology topology = spring.getBean(ID, StormTopology.class);

		Properties systemProperties = System.getProperties();
		if (systemProperties.containsKey(LOCAL_RUN_PARAM)) {
			String timeout = systemProperties.getProperty(LOCAL_RUN_PARAM);
			if (! hasText(timeout))
				timeout = LOCAL_RUN_DEFAULT_TIMEOUT;
			long ms = 1000L * Integer.parseInt(timeout);

			LocalCluster cluster = new LocalCluster();
			cluster.submitTopology(ID, config, topology);
			sleep(ms);
			cluster.shutdown();
		} else
			StormSubmitter.submitTopology(ID, config, topology);
	} catch (Exception e) {
		e.printStackTrace();
		exit(255);
	}
}
 
Example #2
Source File: TypedTupleExample.java    From storm-net-adapter with Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) throws Exception {
    StreamBuilder builder = new StreamBuilder();
    /**
     * The spout emits sequences of (Integer, Long, Long). TupleValueMapper can be used to extract fields
     * from the values and produce a stream of typed tuple (Tuple3<Integer, Long, Long> in this case.
     */
    Stream<Tuple3<Integer, Long, Long>> stream = builder.newStream(new RandomIntegerSpout(), TupleValueMappers.of(0, 1, 2));

    PairStream<Long, Integer> pairs = stream.mapToPair(t -> Pair.of(t._2 / 10000, t._1));

    pairs.window(TumblingWindows.of(Count.of(10))).groupByKey().print();

    String topoName = "test";
    if (args.length > 0) {
        topoName = args[0];
    }
    Config config = new Config();
    config.setNumWorkers(1);
    StormSubmitter.submitTopologyWithProgressBar(topoName, config, builder.build());
}
 
Example #3
Source File: MultiSpoutExclamationTopology.java    From incubator-heron with Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) throws Exception {
  TopologyBuilder builder = new TopologyBuilder();

  builder.setSpout("word0", new TestWordSpout(), 2);
  builder.setSpout("word1", new TestWordSpout(), 2);
  builder.setSpout("word2", new TestWordSpout(), 2);
  builder.setBolt("exclaim1", new ExclamationBolt(), 2)
      .shuffleGrouping("word0")
      .shuffleGrouping("word1")
      .shuffleGrouping("word2");

  Config conf = new Config();
  conf.setDebug(true);
  conf.setMaxSpoutPending(10);
  conf.put(Config.TOPOLOGY_WORKER_CHILDOPTS, "-XX:+HeapDumpOnOutOfMemoryError");
  if (args != null && args.length > 0) {
    conf.setNumWorkers(3);
    StormSubmitter.submitTopology(args[0], conf, builder.createTopology());
  } else {
    LocalCluster cluster = new LocalCluster();
    cluster.submitTopology("test", conf, builder.createTopology());
    Utils.sleep(10000);
    cluster.killTopology("test");
    cluster.shutdown();
  }
}
 
Example #4
Source File: AggregateExample.java    From storm-net-adapter with Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("unchecked")
public static void main(String[] args) throws Exception {
    StreamBuilder builder = new StreamBuilder();
    /**
     * Computes average of the stream of numbers emitted by the spout. Internally the per-partition
     * sum and counts are accumulated and emitted to a downstream task where the partially accumulated
     * results are merged and the final result is emitted.
     */
    builder.newStream(new RandomIntegerSpout(), new ValueMapper<Integer>(0), 2)
           .window(TumblingWindows.of(BaseWindowedBolt.Duration.seconds(5)))
           .filter(x -> x > 0 && x < 500)
           .aggregate(new Avg())
           .print();

    Config config = new Config();
    String topoName = "AGG_EXAMPLE";
    if (args.length > 0) {
        topoName = args[0];
    }
    config.setNumWorkers(1);
    StormSubmitter.submitTopologyWithProgressBar(topoName, config, builder.build());
}
 
Example #5
Source File: AckingTopology.java    From incubator-heron with Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) throws Exception {
  if (args.length != 1) {
    throw new RuntimeException("Specify topology name");
  }
  TopologyBuilder builder = new TopologyBuilder();

  int spouts = 2;
  int bolts = 2;
  builder.setSpout("word", new AckingTestWordSpout(), spouts);
  builder.setBolt("exclaim1", new ExclamationBolt(), bolts)
      .shuffleGrouping("word");

  Config conf = new Config();
  conf.setDebug(true);

  // Put an arbitrary large number here if you don't want to slow the topology down
  conf.setMaxSpoutPending(1000 * 1000 * 1000);

  // To enable acking, we need to setEnableAcking true
  conf.setNumAckers(1);
  conf.put(Config.TOPOLOGY_WORKER_CHILDOPTS, "-XX:+HeapDumpOnOutOfMemoryError");

  // Set the number of workers or stream managers
  conf.setNumWorkers(2);
  StormSubmitter.submitTopology(args[0], conf, builder.createTopology());
}
 
Example #6
Source File: SlidingWindowTopology.java    From incubator-heron with Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) throws Exception {
  TopologyBuilder builder = new TopologyBuilder();
  builder.setSpout("integer", new RandomIntegerSpout(), 1);
  builder.setBolt("slidingsum", new SlidingWindowSumBolt()
      .withWindow(BaseWindowedBolt.Count.of(30), BaseWindowedBolt.Count.of(10)), 1)
      .shuffleGrouping("integer");
  builder.setBolt("tumblingavg", new TumblingWindowAvgBolt()
      .withTumblingWindow(BaseWindowedBolt.Count.of(3)), 1)
      .shuffleGrouping("slidingsum");
  builder.setBolt("printer", new PrinterBolt(), 1)
      .shuffleGrouping("tumblingavg");
  Config conf = new Config();
  conf.setDebug(true);
  String topoName = "test";

  if (args != null && args.length > 0) {
    topoName = args[0];
  }
  StormSubmitter.submitTopology(topoName, conf, builder.createTopology());
}
 
Example #7
Source File: StatefulWindowingTopology.java    From storm-net-adapter with Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) throws Exception {
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("spout", new RandomIntegerSpout());
    builder.setBolt("sumbolt", new WindowSumBolt().withWindow(new Count(5), new Count(3))
                                                  .withMessageIdField("msgid"), 1).shuffleGrouping("spout");
    builder.setBolt("printer", new PrinterBolt(), 1).shuffleGrouping("sumbolt");
    Config conf = new Config();
    conf.setDebug(false);
    //conf.put(Config.TOPOLOGY_STATE_PROVIDER, "org.apache.storm.redis.state.RedisKeyValueStateProvider");

    String topoName = "test";
    if (args != null && args.length > 0) {
        topoName = args[0];
    }
    conf.setNumWorkers(1);
    StormSubmitter.submitTopologyWithProgressBar(topoName, conf, builder.createTopology());
}
 
Example #8
Source File: SentenceWordCountTopology.java    From incubator-heron with Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) throws Exception {
  String name = "fast-word-count-topology";
  if (args != null && args.length > 0) {
    name = args[0];
  }

  TopologyBuilder builder = new TopologyBuilder();

  builder.setSpout("spout", new FastRandomSentenceSpout(), 1);
  builder.setBolt("split", new SplitSentence(), 2).shuffleGrouping("spout");
  builder.setBolt("count", new WordCount(), 2).fieldsGrouping("split", new Fields("word"));

  Config conf = new Config();

  StormSubmitter.submitTopology(name, conf, builder.createTopology());
}
 
Example #9
Source File: MultipleLoggerTopology.java    From storm-net-adapter with Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) throws Exception {
    TopologyBuilder builder = new TopologyBuilder();

    builder.setSpout("word", new TestWordSpout(), 10);
    builder.setBolt("exclaim1", new ExclamationLoggingBolt(), 3).shuffleGrouping("word");
    builder.setBolt("exclaim2", new ExclamationLoggingBolt(), 2).shuffleGrouping("exclaim1");

    Config conf = new Config();
    conf.setDebug(true);
    String topoName = MultipleLoggerTopology.class.getName();
    if (args != null && args.length > 0) {
        topoName = args[0];
    }
    conf.setNumWorkers(2);
    StormSubmitter.submitTopologyWithProgressBar(topoName, conf, builder.createTopology());
}
 
Example #10
Source File: TridentWordCount.java    From storm-net-adapter with Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) throws Exception {
    Config conf = new Config();
    conf.setMaxSpoutPending(20);
    String topoName = "wordCounter";
    if (args.length > 0) {
        topoName = args[0];
    }
    conf.setNumWorkers(3);
    StormSubmitter.submitTopologyWithProgressBar(topoName, conf, buildTopology());
    try (DRPCClient drpc = DRPCClient.getConfiguredClient(conf)) {
        for (int i = 0; i < 10; i++) {
            System.out.println("DRPC RESULT: " + drpc.execute("words", "cat the dog jumped"));
            Thread.sleep(1000);
        }
    }
}
 
Example #11
Source File: TridentMapExample.java    From storm-net-adapter with Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) throws Exception {
    Config conf = new Config();
    conf.setMaxSpoutPending(20);
    String topoName = "wordCounter";
    if (args.length > 0) {
        topoName = args[0];
    }
    conf.setNumWorkers(3);
    StormSubmitter.submitTopologyWithProgressBar(topoName, conf, buildTopology());
    try (DRPCClient drpc = DRPCClient.getConfiguredClient(conf)) {
        for (int i = 0; i < 10; i++) {
            System.out.println("DRPC RESULT: " + drpc.execute("words", "CAT THE DOG JUMPED"));
            Thread.sleep(1000);
        }
    }
}
 
Example #12
Source File: ReachTopology.java    From storm-net-adapter with Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) throws Exception {
    LinearDRPCTopologyBuilder builder = construct();

    Config conf = new Config();
    conf.setNumWorkers(6);
    String topoName = "reach-drpc";
    if (args.length > 0) {
        topoName = args[0];
    }
    StormSubmitter.submitTopologyWithProgressBar(topoName, conf, builder.createRemoteTopology());

    try (DRPCClient drpc = DRPCClient.getConfiguredClient(conf)) {
        String[] urlsToTry = new String[]{ "foo.com/blog/1", "engineering.twitter.com/blog/5", "notaurl.com" };
        for (String url : urlsToTry) {
            System.out.println("Reach of " + url + ": " + drpc.execute("reach", url));
        }
    }
}
 
Example #13
Source File: SlidingWindowTopology.java    From storm-net-adapter with Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) throws Exception {
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("integer", new RandomIntegerSpout(), 1);
    builder.setBolt("slidingsum", new SlidingWindowSumBolt().withWindow(Count.of(30), Count.of(10)), 1)
           .shuffleGrouping("integer");
    builder.setBolt("tumblingavg", new TumblingWindowAvgBolt().withTumblingWindow(Count.of(3)), 1)
           .shuffleGrouping("slidingsum");
    builder.setBolt("printer", new PrinterBolt(), 1).shuffleGrouping("tumblingavg");
    Config conf = new Config();
    conf.setDebug(true);
    String topoName = "test";
    if (args != null && args.length > 0) {
        topoName = args[0];
    }
    conf.setNumWorkers(1);
    StormSubmitter.submitTopologyWithProgressBar(topoName, conf, builder.createTopology());
}
 
Example #14
Source File: SinkTopology.java    From DBus with Apache License 2.0 6 votes vote down vote up
private void start(StormTopology topology, boolean runAsLocal) throws Exception {
    Config conf = new Config();
    conf.put(Constants.ZOOKEEPER_SERVERS, zkConnect);
    conf.put(Constants.TOPOLOGY_ID, topologyId);
    conf.put(Constants.SINK_TYPE, sinkType);
    conf.put(Config.TOPOLOGY_WORKER_CHILDOPTS, sinkerConf.getProperty(SinkerConstants.TOPOLOGY_WORKER_CHILDOPTS));

    conf.setMessageTimeoutSecs(Integer.parseInt(sinkerConf.getProperty(SinkerConstants.STORM_MESSAGE_TIMEOUT)));
    conf.setMaxSpoutPending(Integer.parseInt(sinkerConf.getProperty(SinkerConstants.STORM_MAX_SPOUT_PENDING)));
    conf.setDebug(true);
    conf.setNumWorkers(Integer.parseInt(sinkerConf.getProperty(SinkerConstants.STORM_NUM_WORKERS)));
    if (runAsLocal) {
        conf.setMaxTaskParallelism(10);
        LocalCluster cluster = new LocalCluster();
        cluster.submitTopology(topologyName, conf, topology);
    } else {
        StormSubmitter.submitTopology(topologyName, conf, topology);
    }
}
 
Example #15
Source File: DBusLogProcessorTopology.java    From DBus with Apache License 2.0 6 votes vote down vote up
private void start(StormTopology topology, boolean runAsLocal) throws Exception {
    Config conf = new Config();
    conf.put(com.creditease.dbus.commons.Constants.ZOOKEEPER_SERVERS, zkConnect);
    conf.put(Constants.TOPOLOGY_ID, topologyId);
    conf.setMessageTimeoutSecs(Integer.parseInt(properties.getProperty(Constants.LOG_MESSAGE_TIMEOUT)));
    //conf.setMaxSpoutPending(30);
    conf.setDebug(true);
    conf.setNumWorkers(Integer.parseInt(properties.getProperty(Constants.LOG_NUMWORKERS)));

    if (runAsLocal) {
        conf.setMaxTaskParallelism(10);
        LocalCluster cluster = new LocalCluster();
        cluster.submitTopology(topologyName, conf, topology);
    } else {
        StormSubmitter.submitTopology(topologyName, conf, topology);
    }
}
 
Example #16
Source File: FullPullerTopology.java    From DBus with Apache License 2.0 6 votes vote down vote up
private void start(StormTopology topology, boolean runAsLocal) throws Exception {
    Config conf = new Config();
    conf.put(FullPullConstants.FULL_SPLITTER_TOPOLOGY_ID, fullSplitterTopologyId);
    conf.put(FullPullConstants.FULL_PULLER_TOPOLOGY_ID, fullPullerTopologyId);
    conf.put(FullPullConstants.DS_NAME, topologyId);
    conf.put(FullPullConstants.ZKCONNECT, zkConnect);
    conf.put(Config.TOPOLOGY_WORKER_CHILDOPTS, commonConfSplit.getProperty(FullPullConstants.TOPOLOGY_WORKER_CHILDOPTS));
    //设置message超时时间为,保证每个分片都能在该内拉完数据
    conf.setMessageTimeoutSecs(Integer.parseInt(commonConfSplit.getProperty(FullPullConstants.STORM_MESSAGE_TIMEOUT)));
    conf.setMaxSpoutPending(Integer.parseInt(commonConfSplit.getProperty(FullPullConstants.STORM_MAX_SPOUT_PENDING)));
    conf.setNumWorkers(Integer.parseInt(commonConfSplit.getProperty(FullPullConstants.STORM_NUM_WORKERS)));
    conf.setDebug(true);

    if (runAsLocal) {
        conf.setMaxTaskParallelism(3);
        LocalCluster cluster = new LocalCluster();
        cluster.submitTopology(topologyName, conf, topology);
    } else {
        StormSubmitter.submitTopology(topologyName, conf, topology);
    }
}
 
Example #17
Source File: SlidingTupleTsTopology.java    From storm-net-adapter with Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) throws Exception {
    TopologyBuilder builder = new TopologyBuilder();
    BaseWindowedBolt bolt = new SlidingWindowSumBolt()
        .withWindow(new Duration(5, TimeUnit.SECONDS), new Duration(3, TimeUnit.SECONDS))
        .withTimestampField("ts")
        .withLag(new Duration(5, TimeUnit.SECONDS));
    builder.setSpout("integer", new RandomIntegerSpout(), 1);
    builder.setBolt("slidingsum", bolt, 1).shuffleGrouping("integer");
    builder.setBolt("printer", new PrinterBolt(), 1).shuffleGrouping("slidingsum");
    Config conf = new Config();
    conf.setDebug(true);
    String topoName = "test";

    if (args != null && args.length > 0) {
        topoName = args[0];
    }

    conf.setNumWorkers(1);
    StormSubmitter.submitTopologyWithProgressBar(topoName, conf, builder.createTopology());
}
 
Example #18
Source File: WordCountTopologyNode.java    From storm-net-adapter with Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) throws Exception {

        TopologyBuilder builder = new TopologyBuilder();

        builder.setSpout("spout", new RandomSentence(), 5);

        builder.setBolt("split", new SplitSentence(), 8).shuffleGrouping("spout");
        builder.setBolt("count", new WordCount(), 12).fieldsGrouping("split", new Fields("word"));

        Config conf = new Config();
        conf.setDebug(true);
        String topoName = "word-count";
        if (args != null && args.length > 0) {
            topoName = args[0];
        }
        conf.setNumWorkers(3);
        StormSubmitter.submitTopologyWithProgressBar(topoName, conf, builder.createTopology());
    }
 
Example #19
Source File: DrpcTestTopologyCsharp.java    From storm-net-adapter with Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) throws Exception {
  	TopologyBuilder builder = new TopologyBuilder();
	  
  	DRPCSpout drpcSpout = new DRPCSpout("simpledrpc");
    builder.setSpout("drpc-input", drpcSpout,1);

    builder.setBolt("simple", new SimpleDRPC(), 2)
    		.noneGrouping("drpc-input");
    
    builder.setBolt("return", new ReturnResults(),1)
	.noneGrouping("simple");

    Config conf = new Config();
    conf.setDebug(true);
    conf.setMaxTaskParallelism(1);
    
    try
    {
    	StormSubmitter.submitTopology("drpc-q", conf,builder.createTopology());
    }
    catch (Exception e)
    {
    	e.printStackTrace();
    }
}
 
Example #20
Source File: StreamToEs.java    From elasticsearch-hadoop with Apache License 2.0 6 votes vote down vote up
public static void submitJob(String principal, String keytab, String esNodes) throws Exception {
    List doc1 = Collections.singletonList("{\"reason\" : \"business\",\"airport\" : \"SFO\"}");
    List doc2 = Collections.singletonList("{\"participants\" : 5,\"airport\" : \"OTP\"}");

    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("Input", new TestSpout(ImmutableList.of(doc1, doc2), new Fields("json"), true));
    builder.setBolt("ES", new EsBolt("storm-test"))
            .shuffleGrouping("Input")
            .addConfiguration(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS, 2);

    // Nimbus needs to be started with the cred renewer and credentials plugins set in its config file

    Config conf = new Config();
    List<Object> plugins = new ArrayList<Object>();
    plugins.add(AutoElasticsearch.class.getName());
    conf.put(Config.TOPOLOGY_AUTO_CREDENTIALS, plugins);
    conf.put(ConfigurationOptions.ES_NODES, esNodes);
    conf.put(ConfigurationOptions.ES_SECURITY_AUTHENTICATION, "kerberos");
    conf.put(ConfigurationOptions.ES_NET_SPNEGO_AUTH_ELASTICSEARCH_PRINCIPAL, "HTTP/[email protected]");
    conf.put(ConfigurationOptions.ES_INPUT_JSON, "true");
    StormSubmitter.submitTopology("test-run", conf, builder.createTopology());
}
 
Example #21
Source File: StreamFromEs.java    From elasticsearch-hadoop with Apache License 2.0 6 votes vote down vote up
public static void submitJob(String principal, String keytab, String esNodes) throws Exception {
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("ES", new EsSpout("storm-test"));
    builder.setBolt("Output", new CapturingBolt()).shuffleGrouping("ES");

    // Nimbus needs to be started with the cred renewer and credentials plugins set in its config file

    Config conf = new Config();
    List<Object> plugins = new ArrayList<Object>();
    plugins.add(AutoElasticsearch.class.getName());
    conf.put(Config.TOPOLOGY_AUTO_CREDENTIALS, plugins);
    conf.put(ConfigurationOptions.ES_NODES, esNodes);
    conf.put(ConfigurationOptions.ES_SECURITY_AUTHENTICATION, "kerberos");
    conf.put(ConfigurationOptions.ES_NET_SPNEGO_AUTH_ELASTICSEARCH_PRINCIPAL, "HTTP/[email protected]");
    conf.put(ConfigurationOptions.ES_INPUT_JSON, "true");
    StormSubmitter.submitTopology("test-read", conf, builder.createTopology());
}
 
Example #22
Source File: ConfigurableTopology.java    From storm-crawler with Apache License 2.0 6 votes vote down vote up
/** Submits the topology under a specific name **/
protected int submit(String name, Config conf, TopologyBuilder builder) {

    // register Metadata for serialization with FieldsSerializer
    Config.registerSerialization(conf, Metadata.class);

    if (isLocal) {
        LocalCluster cluster = new LocalCluster();
        cluster.submitTopology(name, conf, builder.createTopology());
        if (ttl != -1) {
            Utils.sleep(ttl * 1000);
            cluster.shutdown();
        }
    }

    else {
        try {
            StormSubmitter.submitTopology(name, conf,
                    builder.createTopology());
        } catch (Exception e) {
            e.printStackTrace();
            return -1;
        }
    }
    return 0;
}
 
Example #23
Source File: StatisticTopology.java    From storm-statistic with Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) throws Exception {
    TopologyBuilder builder = new TopologyBuilder();
    /**
     * 设置spout和bolt的dag(有向无环图)
     */
    KafkaSpout kafkaSpout = createKafkaSpout();
    builder.setSpout("id_kafka_spout", kafkaSpout);
    builder.setBolt("id_convertIp_bolt", new ConvertIPBolt()).shuffleGrouping("id_kafka_spout"); // 通过不同的数据流转方式,来指定数据的上游组件
    builder.setBolt("id_statistic_bolt", new StatisticBolt()).shuffleGrouping("id_convertIp_bolt"); // 通过不同的数据流转方式,来指定数据的上游组件
    // 使用builder构建topology
    StormTopology topology = builder.createTopology();
    String topologyName = KafkaStormTopology.class.getSimpleName();  // 拓扑的名称
    Config config = new Config();   // Config()对象继承自HashMap,但本身封装了一些基本的配置

    // 启动topology,本地启动使用LocalCluster,集群启动使用StormSubmitter
    if (args == null || args.length < 1) {  // 没有参数时使用本地模式,有参数时使用集群模式
        LocalCluster localCluster = new LocalCluster(); // 本地开发模式,创建的对象为LocalCluster
        localCluster.submitTopology(topologyName, config, topology);
    } else {
        StormSubmitter.submitTopology(topologyName, config, topology);
    }
}
 
Example #24
Source File: PerformanceTestTopology.java    From jstorm with Apache License 2.0 6 votes vote down vote up
public static void SetRemoteTopology()
        throws Exception {
    String streamName = (String) conf.get(Config.TOPOLOGY_NAME);
    if (streamName == null) {
        String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\.");
        streamName = className[className.length - 1];
    }
    
    TopologyBuilder builder = new TopologyBuilder();
    
    int spout_Parallelism_hint = Utils.getInt(conf.get(TOPOLOGY_SPOUT_PARALLELISM_HINT), 1);
    int bolt_Parallelism_hint = Utils.getInt(conf.get(TOPOLOGY_BOLT_PARALLELISM_HINT), 2);
    builder.setSpout("spout", new TestSpout(), spout_Parallelism_hint);
    
    BoltDeclarer boltDeclarer = builder.setBolt("bolt", new TestBolt(), bolt_Parallelism_hint);
    // localFirstGrouping is only for jstorm
    // boltDeclarer.localFirstGrouping(SequenceTopologyDef.SEQUENCE_SPOUT_NAME);
    boltDeclarer.shuffleGrouping("spout");
    // .addConfiguration(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS, 60);
    
    StormSubmitter.submitTopology(streamName, conf, builder.createTopology());
    
}
 
Example #25
Source File: TridentHBaseWindowingStoreTopology.java    From storm-net-adapter with Apache License 2.0 5 votes vote down vote up
public static void main(String[] args) throws Exception {
    Config conf = new Config();
    conf.setMaxSpoutPending(20);
    conf.put(Config.TOPOLOGY_TRIDENT_WINDOWING_INMEMORY_CACHE_LIMIT, 100);

    // window-state table should already be created with cf:tuples column
    HBaseWindowsStoreFactory windowStoreFactory =
        new HBaseWindowsStoreFactory(new HashMap<String, Object>(), "window-state", "cf".getBytes("UTF-8"), "tuples".getBytes("UTF-8"));
    String topoName = "wordCounterWithWindowing";
    if (args.length > 0) {
        topoName = args[0];
    }
    conf.setNumWorkers(3);
    StormSubmitter.submitTopologyWithProgressBar(topoName, conf, buildTopology(windowStoreFactory));
}
 
Example #26
Source File: BasicDRPCTopology.java    From storm-net-adapter with Apache License 2.0 5 votes vote down vote up
public static void main(String[] args) throws Exception {
    Config conf = new Config();
    String topoName = "DRPCExample";
    String function = "exclamation";
    if (args != null) {
        if (args.length > 0) {
            topoName = args[0];
        }
        if (args.length > 1) {
            function = args[1];
        }
    }

    LinearDRPCTopologyBuilder builder = new LinearDRPCTopologyBuilder(function);
    builder.addBolt(new ExclaimBolt(), 3);

    conf.setNumWorkers(3);
    StormSubmitter.submitTopologyWithProgressBar(topoName, conf, builder.createRemoteTopology());

    if (args != null && args.length > 2) {
        try (DRPCClient drpc = DRPCClient.getConfiguredClient(conf)) {
            for (int i = 2; i < args.length; i++) {
                String word = args[i];
                System.out.println("Result for \"" + word + "\": " + drpc.execute(function, word));
            }
        }
    }
}
 
Example #27
Source File: ManualDRPC.java    From storm-net-adapter with Apache License 2.0 5 votes vote down vote up
public static void main(String[] args) throws Exception {
    TopologyBuilder builder = new TopologyBuilder();

    DRPCSpout spout = new DRPCSpout("exclamation");
    builder.setSpout("drpc", spout);
    builder.setBolt("exclaim", new ExclamationBolt(), 3).shuffleGrouping("drpc");
    builder.setBolt("return", new ReturnResults(), 3).shuffleGrouping("exclaim");

    Config conf = new Config();
    StormSubmitter.submitTopology("exclaim", conf, builder.createTopology());
    try (DRPCClient drpc = DRPCClient.getConfiguredClient(conf)) {
        System.out.println(drpc.execute("exclamation", "aaa"));
        System.out.println(drpc.execute("exclamation", "bbb"));
    }
}
 
Example #28
Source File: BranchExample.java    From storm-net-adapter with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
public static void main(String[] args) throws Exception {
    StreamBuilder builder = new StreamBuilder();
    Stream<Integer>[] evenAndOdd = builder
        /*
         * Create a stream of random numbers from a spout that
         * emits random integers by extracting the tuple value at index 0.
         */
        .newStream(new RandomIntegerSpout(), new ValueMapper<Integer>(0))
        /*
         * Split the stream of numbers into streams of
         * even and odd numbers. The first stream contains even
         * and the second contains odd numbers.
         */
        .branch(x -> (x % 2) == 0,
                x -> (x % 2) == 1);

    evenAndOdd[0].forEach(x -> LOG.info("EVEN> " + x));
    evenAndOdd[1].forEach(x -> LOG.info("ODD > " + x));

    Config config = new Config();
    String topoName = "branchExample";
    if (args.length > 0) {
        topoName = args[0];
    }

    config.setNumWorkers(1);
    StormSubmitter.submitTopologyWithProgressBar(topoName, config, builder.build());
}
 
Example #29
Source File: PersistentWindowingTopology.java    From storm-net-adapter with Apache License 2.0 5 votes vote down vote up
/**
 * Create and deploy the topology.
 *
 * @param args args
 * @throws Exception exception
 */
public static void main(String[] args) throws Exception {
    TopologyBuilder builder = new TopologyBuilder();

    // generate random numbers
    builder.setSpout("spout", new RandomIntegerSpout());

    // emits sliding window and global averages
    builder.setBolt("avgbolt", new AvgBolt()
        .withWindow(new Duration(10, TimeUnit.SECONDS), new Duration(2, TimeUnit.SECONDS))
        // persist the window in state
        .withPersistence()
        // max number of events to be cached in memory
        .withMaxEventsInMemory(25000), 1)
           .shuffleGrouping("spout");

    // print the values to stdout
    builder.setBolt("printer", (x, y) -> System.out.println(x.getValue(0)), 1).shuffleGrouping("avgbolt");

    Config conf = new Config();
    conf.setDebug(false);

    // checkpoint the state every 5 seconds
    conf.put(Config.TOPOLOGY_STATE_CHECKPOINT_INTERVAL, 5000);

    // use redis for state persistence
    conf.put(Config.TOPOLOGY_STATE_PROVIDER, "org.apache.storm.redis.state.RedisKeyValueStateProvider");

    String topoName = "test";
    if (args != null && args.length > 0) {
        topoName = args[0];
    }
    conf.setNumWorkers(1);
    StormSubmitter.submitTopologyWithProgressBar(topoName, conf, builder.createTopology());
}
 
Example #30
Source File: StatefulTopology.java    From storm-net-adapter with Apache License 2.0 5 votes vote down vote up
public static void main(String[] args) throws Exception {
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("spout", new RandomIntegerSpout());
    builder.setBolt("partialsum", new StatefulSumBolt("partial"), 1).shuffleGrouping("spout");
    builder.setBolt("printer", new PrinterBolt(), 2).shuffleGrouping("partialsum");
    builder.setBolt("total", new StatefulSumBolt("total"), 1).shuffleGrouping("printer");
    Config conf = new Config();
    conf.setDebug(false);
    String topoName = "test";
    if (args != null && args.length > 0) {
        topoName = args[0];
    }
    conf.setNumWorkers(1);
    StormSubmitter.submitTopologyWithProgressBar(topoName, conf, builder.createTopology());
}